diff --git a/.travis.yml b/.travis.yml index e072a9ed6..c09ad9831 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,4 +50,4 @@ install: - pip3 install . - pip3 install pytest - pip3 install pydarnio -script: pytest --ignore=test/integration +script: pytest --ignore=test/data diff --git a/.zenodo.json b/.zenodo.json index 906b0df1b..483724eae 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -12,30 +12,20 @@ "name": "Tholley, F." }, { - "affiliation": "University of Scranton", - "name": "Frissell, N.", - "orcid": "0000-0002-8398-4222" + "affiliation": "University of Saskatchewan", + "name": "Martin, C.J.", + "orcid": "0000-0002-8278-9783" }, { "affiliation": "University of Saskatchewan", "name": "Billett, D.D.", "orcid": "0000-0002-8905-8609" }, - { - "affiliation": "University of Saskatchewan", - "name": "Martin, C.J.", - "orcid": "0000-0002-8278-9783" - }, { "affiliation": "The University Centre in Svalbard", "name": "Bland, E.C.", "orcid": "0000-0002-0252-0400" }, - { - "affiliation": "US Naval Research Laboratory", - "name": "Burrell, A.G.", - "orcid": "0000-0001-8875-9326" - }, { "affiliation": "Virginia Tech", "name": "Coyle, S.", @@ -45,6 +35,11 @@ "affiliation": "University of Saskatchewan", "name": "Detwiller, M.H." }, + { + "affiliation": "University of Scranton", + "name": "Frissell, N.", + "orcid": "0000-0002-8398-4222" + }, { "affiliation": "The University Centre in Svalbard", "name": "Herlingshaw, K.", @@ -55,11 +50,6 @@ "name": "Huyghebaert, D.", "orcid": "0000-0002-4257-4235" }, - { - "affiliation": "SRI International", - "name": "Reimer, A.S.", - "orcid": "0000-0002-4621-3453" - }, { "affiliation": "University of Bergen", "name": "Reistad, J.P.", diff --git a/MANIFEST.in b/MANIFEST.in index 8b37a7d8a..97f6e4c18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,5 +10,3 @@ include pydarn/logging_config.yaml # Include hardware files include pydarn/utils/hdw/* -#Include radar FOV files -include pydarn/radar_fov_files/* diff --git a/README.md b/README.md index 97ec6d94a..8135cc327 100644 --- a/README.md +++ b/README.md @@ -7,24 +7,22 @@ Python data visualization library for the Super Dual Auroral Radar Network (SuperDARN). -**WARNING!** - This is the `develop` branch that is not DOI'd yet, please click on the above DOI to get the most recently released pyDARN version. - ## Changelog -## Version 2.1 - Release! +## Version 2.2 - Release! +**New Requirement**: pyDARN 2.2 requires minimum matplotlib version of 3.3.4 -pyDARN release v2.1 includes the following features: -- Updates to Fan plots -- Updates to FOV plots -- **NEW** Grid Plot -- **NEW** Ground-Scatter Mapped Range option for Range-Time plots -- **NEW** `Coords` enumerator for selecting coordinate system -- updated documentation +pyDARN release v2.2 includes the following features: +- **New** Radar geographical Field-of-View (FOV) calculations functionality +- **Deprecation** Removed FOV files for radars geographic beam location +- **New** pytesting environment for testing pydarn for development +- **New** FOV grid lines to show range gates +- **New** `channel` option in Fan plots +- Bug fix in Range-time plots with groundscatter +- Bug fix in Range-time plots with time-shifting -!!! Warning - `slant` option in `plot_range-time` and `plot_summary` is deprecated now uses `coords` +**Deprecation**: `slant` option in `plot_range-time` and `plot_summary` is deprecated now uses `coords` ## Documentation @@ -66,7 +64,7 @@ For more information and tutorials on pyDARN please see the [tutorial section](h pyDARN is always looking for testers and developers keen on learning python, github, and/or SuperDARN data visualizations! Here are some ways to get started: - - **Testing Pull Request**: to determine which [pull requests](https://github.com/SuperDARN/pydarn/pulls) need to be tested right away, filter them by their milestones (v1.1.0 is currently highest priority). + - **Testing Pull Request**: to determine which [pull requests](https://github.com/SuperDARN/pydarn/pulls) need to be tested right away, filter them by their milestones (v3.0 is currently highest priority). - **Getting involved in projects**: if you are looking to help in a specific area, look at pyDARN's [projects tab](https://github.com/SuperDARN/pydarn/projects). The project you are interested in will give you information on what is needed to reach completion. This includes things currently in progress, and those awaiting reviews. - **Answer questions**: if you want to try your hand at answering some pyDARN questions, or adding to the discussion, look at pyDARN's [issues](https://github.com/SuperDARN/pydarn/issues) and filter by labels. - **Become a developer**: if you want to practice those coding skills and add to the library, look at pyDARN [issues](https://github.com/SuperDARN/pydarn/issues) and filter by milestone's to see what needs to get done right away. diff --git a/docs/dev/PR.md b/docs/dev/PR.md index 3a520a50d..6119f556b 100644 --- a/docs/dev/PR.md +++ b/docs/dev/PR.md @@ -24,6 +24,8 @@ Before submitting a PR check the code check list first: - copyright and disclaimer, or modification line is added - the code installs with python 3.6 and higher +- the code doesn't fail on [`pytest` test](pytest.md) +- any new features need to have [tests for `pytest` to run on](pytest.md) - documentation is updated or in another PR for users - Follows [PEP8](https://www.python.org/dev/peps/pep-0008/) code style and [pandas docstring](https://pandas.pydata.org/pandas-docs/stable/development/contributing_docstring.html) style - unit tests or various testing has been added/done diff --git a/docs/dev/branching.md b/docs/dev/branching.md index 9b874ae94..e66ab77b0 100644 --- a/docs/dev/branching.md +++ b/docs/dev/branching.md @@ -66,4 +66,4 @@ In git, "branches" are used to make changes to the code without affecting the ma git push origin * Repeat the above commands above as you work on the code changes -* Once you have completed, documented, and tested your code then you can create a pull request, see [pull request](PR.md) +* Once you have completed, documented, tested and updated the [unit tests](pytest.md) then you can create a pull request, see [pull request](PR.md) diff --git a/docs/dev/communication.md b/docs/dev/communication.md index 37a29816b..0f02dfa0c 100644 --- a/docs/dev/communication.md +++ b/docs/dev/communication.md @@ -1,6 +1,6 @@ # Communication Guidelines -pyDARN is under the Data Analysis Working Group that has set up guidelines on [communication](https://superdarn.github.io/dawg/documents/communication-guidelines/) to encourage a positive collaborative environment. +pyDARN is under the Data Analysis Working Group that has set up guidelines on [communication](communication.md) to encourage a positive collaborative environment. Please follow these guidelines when commenting and reviewing code on GitHub. Also please read workflow documentation on issues and [pull requests](PR.md) to communicate effectively for collaborative work. @@ -10,4 +10,4 @@ Also please read workflow documentation on issues and [pull requests](PR.md) to If members have a dispute that violates the communication guidelines, and creates a negative environment in the pyDARN community, please follow this procedure to resolve the conflict: 1. Please try resolve the matter between each member offline. 2. If the dispute cannot be resolved, or the environment has become toxic and overwhelming, please contact the [lead developer for pyDARN](https://pydarn.readthedocs.io/en/latest/). -3. If the dispute cannot be resolved, involves the lead developer, or breaches the [DAWG charter](https://superdarn.github.io/dawg/documents/DAWG_Charter/) please contact the [chairs of DAWG](https://superdarn.github.io/dawg/about/). +3. If the dispute cannot be resolved, involves the lead developer, or breaches the [DVWG charter](https://superdarn.github.io/DVWG/docs/charter/) please contact the [chair of ](https://superdarn.github.io/DVWG/about/). diff --git a/docs/dev/issues.md b/docs/dev/issues.md index 332fc5301..e24578332 100644 --- a/docs/dev/issues.md +++ b/docs/dev/issues.md @@ -34,7 +34,7 @@ Please only open issues related to pyDARN. Questions on the following topics sho - Corrupt or missing RAWACF files: [Data Distribution Working Group (DDWG)](https://github.com/SuperDARN/DDWG) - Radar data formats, hardware files and other metadata: [Data Standards Working Group (DSWG)](https://github.com/superdarn/dswg-published-docs) - Radar control programs and operating schedules: [Scheduling Working Group (SWG)](http://superdarn.thayer.dartmouth.edu/wg-scd.html) -- General questions for the Data Analysis Working Group (DAWG) that are not specifically about pyDARN: open an issue on the [internal DAWG Github repo](https://github.com/SuperDARN/dawg) (requires access rights) or email the [DAWG mailing list](mailto:darn-dawg@isee.nagoya-u.ac.jp) +- General questions for the Data Visualization Working Group (DVWG) that are not specifically about pyDARN: open an issue on the [internal DVWG Github repo](https://github.com/SuperDARN/DVWG) (requires access rights) or email the [DVWG mailing list](mailto:darn-dvwg@isee.nagoya-u.ac.jp) ## Submit an Issue @@ -49,7 +49,7 @@ To open a new issue, follow these steps from the [pyDARN repository](https://git - Assignees: Assign yourself to the issue if you are planning to work on it yourself. Leave blank if you are not planning to work on it. - Labels: these are used to categorize and filter issues (examples below). You can have multiple labels. - Projects: Indicate if this issue is part of a [project](projects.md) - - Milestone: Please leave this blank. The DAWG chairs will fill in this information later when a timeline emerges (e.g. working towards a new release) + - Milestone: Please leave this blank. The DVWG chairs will fill in this information later when a timeline emerges (e.g. working towards a new release) - Linked Pull Request: Leave this blank for now. This can be filled out when a pull request that solves this issue is created (see [pull requests](PR.md)) * Click *Submit new issue*. At this point, everyone who is subscribed to the pyDARN repository will receive an email. Depending on your notification settings, you may receive emails when others comment on the issue. Check your user settings to subscribe or unsubscribe to email notifications. @@ -57,9 +57,9 @@ To open a new issue, follow these steps from the [pyDARN repository](https://git ## Following The Issue Please follow the ongoing conversation about your issue, especially if other developers request more information. -If you do not respond for a while, the issue will go **stale** and may be closed. DAWG tries to keep issues open and contact users if their issues have gone stale. +If you do not respond for a while, the issue will go **stale** and may be closed. DVWG tries to keep issues open and contact users if their issues have gone stale. -Please keep in mind that DAWG members have priorities other than pyDARN, and they live in many different time zones, so be patient while waiting for a response. If nobody has responded in 1-2 weeks, it's fine to ask again. +Please keep in mind that DVWG members have priorities other than pyDARN, and they live in many different time zones, so be patient while waiting for a response. If nobody has responded in 1-2 weeks, it's fine to ask again. ## Closing Issues @@ -76,10 +76,10 @@ If an issue goes stale (no progress for over 3 months), the pyDARN lead develope 1. Ask about the issue's status in the conversation 2. Email the user who opened the issue -3. Request that we discuss the issue in the next DAWG meeting (add label *telecon*), or place it in a project to be re-opened later +3. Request that we discuss the issue in the next DVWG meeting (add label *telecon*), or place it in a project to be re-opened later * Label the issue **stale** until progress can be made on it again * Request to close the issue if none of the above steps elicit any information to keep it open -* DAWG chairs can close **stale** issues +* DVWG chairs can close **stale** issues ## Assignees @@ -93,7 +93,7 @@ pyDARN supports the following [labels](https://github.com/SuperDARN/pydarn/label | Label | Definition | | ----------------- | ----------------------------------------------------------- | | bug | Issue is about a potential bug | -| DAWG | Issue for Data Analysis Working Group | +| DVWG | Issue for Data Analysis Working Group | | DDWG | Issue for Data Distribution Working Group | | default change | Default change in one of the methods | | Deprecation | Removing legacy code/feature from the codebase | @@ -132,5 +132,5 @@ Then click on *Create Label*. ## Open-ended Issues Sometimes Issues can be open-ended, especially discussion issues. Here are some steps to make progress on the issue and prevent it going **stale**: -- Label it with *telecon* if needs to be further discussed in a telecon with DAWG members +- Label it with *telecon* if needs to be further discussed in a telecon with DVWG members - Once a general idea is established, close the issue and create a project to break it into a defined tasks. Open the first task as an issue, see [projects](projects.md). diff --git a/docs/dev/pytest.md b/docs/dev/pytest.md new file mode 100644 index 000000000..6dcd95a77 --- /dev/null +++ b/docs/dev/pytest.md @@ -0,0 +1,61 @@ + + +## Unit Testing + +Unit testing in pyDARN is a way to check that pydarn is running correctly with any new changes to the code. These tests are meant to be simple and short to only test input (parameter) changes to ensure the code does not fail. +Exceptions can also be tested to make sure it fails correctly. Please note testing in pyDARN is for validity (code runs as it should) but not validation (outputting accurate plots). Validation testing is done by users who understand the scientific accuracy the plots represent. + +pyDARN uses [`pytest`](https://docs.pytest.org/en/6.2.x/) for unit testing. + +## Using `pytest` + +Running `pytest` in pyDARN is for developers and testers that work with GitHub. To run `pytest` in pyDARN: + +!!! Warning + Make sure you install pytest in your virtual environment or on your computer + +1. [Clone the pydarn repository](https://github.com/superdarn/pydarn) +2. Change directory to pyDARN `cd pydarn` +3. Install pydarn `pip install . --user` + - you can install a [virtual environment](../users/install.md) first to protect your current pyDARN install +4. Now run pytest `pytest` it should report no fails +5. If it reports a fail please look into it if you are the developer of the change, or report it on the pull request if you are testing + +## Writing pytest tests + +### Adding to Pre-existing Tests + +Changing, adding, and/or removing parameters from pyDARN methods, means the test code will need to change in the `parameterize` decorator. Please refer to the `test_.py` file that corresponds with the methods module you changed. + +- If you **changed a parameters input type, limitation, or name**, then look for the parameter in the `parameterize` decorators list (typically above the class) and changed the `key` name if you changed the parameter name or `value` if the type or limits are changed. Then update the method calls that take the parameter. +- If you **add a new parameter** to a pyDARN method, then add a `parameterize` decorator before the class (with the other ones) and add the parameter name as the `key` name and add potential `value` test cases. (Not too many as this will make it slow to test). Make sure to add it to the corresponding method tests as well. +- If you **remove a parameter** then remove the `parameterize` decorator corresponding to the parameter. + +### Adding a New Test Module + +If you are creating a new class or module (new python file) then you will need to add a new test module (file) to the `pydarn/test` directory. +Name the file test_.py such that `pytest` can find it and run it. + +You can use `test/test_rtp.py` as a template on how to write test code and read [`pytest`](https://docs.pytest.org/en/6.2.x/) on other plugins to for testing if its needed. + +Make sure to add what is needed to test and not too many parameterization options as it will make the tests go slow. Keep it to bare minimum needed to the test the parameter input. +Remember this is not to test the accuracy as pytest cannot look at the plot but rather to test edge cases of parameters that make it fail. + +!!! Note + There are files provided for testing in the `test/data/` directory. If you need to add a file, please keep the file size as small as possible and check in with other developers before adding. If it's too big, it may not be able to be added to GitHub and cause pydarn to be slow to download. + +To avoid writing several tests with the same body of code, consider using the `parameterize` to provide various types of inputs for one function. + +If you want to reuse `parameterize` parameters for other tests place them in a class, all methods of that class have to take the `parameterize` parameters. One can add more parameters to a method by placing the `parameterize` decorator and parameter name above it and then include in the methods input. diff --git a/docs/dev/releases.md b/docs/dev/releases.md index 45ebc9513..72db2b567 100644 --- a/docs/dev/releases.md +++ b/docs/dev/releases.md @@ -67,7 +67,7 @@ and to confirm the version number is correct. State why a release is needed and 5. Add the following to the PR info: - **Title**: Release pyDARN - **Scope**: add information on the main focus of this release - - **Include link to readthedocs** on this release (ask for pyDARN lead developer or DAWG Chair to do this) + - **Include link to readthedocs** on this release (DVWG Chair to do this) - **Changes:** history of what has changed, look at the [milestones closed tasks](https://github.com/SuperDARN/pydarn/milestones?state=open) - **Approvals:** 3 (reviews+testing) - **Testing:** Using Github Checklists to include what needs to be tested, tested on what kind of data, and operating systems @@ -90,14 +90,14 @@ and to confirm the version number is correct. State why a release is needed and !!! IMPORTANT Do this before the pull request to make it part of the review process -- Make sure the `.zenodo.json` file is updated to reflect the [DAWG authorship guidelines]() and everyone is okay with the order. Please confirm with additional new members if their name and ORCID iD is correct. +- Make sure the `.zenodo.json` file is updated to reflect the [DVWG authorship guidelines]() and everyone is okay with the order. Please confirm with additional new members if their name and ORCID iD is correct. - Update `setup.py` file to have the new version number on the line `version=""` - Update `README.md` file to reflect the new changes of the release and version number ### Release Checklist Once the test is complete and at least three approvals are obtained make sure the following steps are performed. -The pyDARN lead developer or DAWG chairs should do this step; however, if you request to do it and get approval then go ahead! +The pyDARN lead developer or DVWG chair should do this step; however, if you request to do it and get approval then go ahead! 1. **merge** the release branch in `main` and confirm the above updates are there 2. [Tag and release the code](https://github.com/SuperDARN/pydarn/releases/new) @@ -106,7 +106,7 @@ The pyDARN lead developer or DAWG chairs should do this step; however, if you re - Release Title: pyDARN v - Description: Header "Version *number* - Release!" then add "pyDARN release v*number* includes the following features:" listing all the new changes to the code. Please see other [releases](https://github.com/SuperDARN/pydarn/releases) to keep with consistency - Hit Publish Release! -- Once a release has been made check on the [pyDARN Zenodo](https://zenodo.org/record/3727269) and look for the version you just released on the right hand side under **Versions**. Please note this may take some time. If this does not work Contact Lead Developer and DAWG Chairs on the matter. +- Once a release has been made check on the [pyDARN Zenodo](https://zenodo.org/record/3727269) and look for the version you just released on the right hand side under **Versions**. Please note this may take some time. If this does not work Contact Lead Developer and DVWG Chair on the matter. - Once the Zenodo DOI is made for the new release, select the DOI markdown tag on the right hand side below **publication date**. Copy the markdown syntax. - `git checkout main` and `git pull origin main` - Paste this syntax in `docs/user/citing.md` under DOI's. @@ -167,7 +167,7 @@ This step requires the lead developers help as they have access to the pyDARN Py After the above is done do the following to make sure everything is up to date: -- Let the DAWG chairs know a pyDARN release has been made to update the DAWG website +- Let the DVWG chair know a pyDARN release has been made to update the DVWG website - Email [DARN-users](darn-users@isee.nagoya-u.ac.jp) about the new release of pyDARN - Merge `main` back into `develop`: diff --git a/docs/dev/team.md b/docs/dev/team.md index 7ba068092..efa5d11dd 100644 --- a/docs/dev/team.md +++ b/docs/dev/team.md @@ -1,10 +1,9 @@ # pyDARN Team -pyDARN is an open source code developed by the SuperDARN community for data visualization. The library resides under the [Data Analysis Working Group (DAWG)](https://superdarn.github.io/dawg/) with support and keeping up data analysis software. +pyDARN is an open source code developed by the SuperDARN community for data visualization. The library resides under the [Data Visualization Working Group (DVWG)](https://superdarn.github.io/DVWG/) with support and keeping up data analysis software. If you wish to join the pyDARN Team please do the following: -1. Email the [lead developer](https://superdarn.github.io/dawg/about/) to be able to contribute to the [pyDARN GitHub Repository](https://github.com/superdarn/pydarn). -2. Fill out the [DAWG survey](https://docs.google.com/forms/d/e/1FAIpQLScXAupACBEWFQbyq1Y_A9uUPLT-3_nMXdE9RCudscNdzDvcnw/viewform?usp=sf_link) for us to better get to know you and how you would like to contribute. -3. Sign up for the [DAWG emailing list](https://portal.isee.nagoya-u.ac.jp:8443/mailman/listinfo/darn-dawg) for updates in DAWG and upcoming meetings including pyDARN matters. +1. Email the [lead developer](https://superdarn.github.io/DVWG/about/) to be able to contribute to the [pyDARN GitHub Repository](https://github.com/superdarn/pydarn). +3. Sign up for the [DVWG emailing list](https://portal.isee.nagoya-u.ac.jp:8443/mailman/listinfo/darn-dvwg) for updates in DVWG and upcoming meetings including pyDARN matters. 4. Please read the [developer documentation](https://pydarn.readthedocs.io/en/latest/) to ensure you follow guidelines. diff --git a/docs/imgs/pwr0_range-time.png b/docs/imgs/pwr0_range-time.png new file mode 100644 index 000000000..54d5064d2 Binary files /dev/null and b/docs/imgs/pwr0_range-time.png differ diff --git a/docs/index.md b/docs/index.md index cd6ad564d..b454fe399 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,7 @@ If you have any questions or concerns please submit an **Issue** on the SuperDAR - [Projects](dev/projects.md) - [Issues](dev/issues.md) - [Branches](dev/branching.md) + - [Unit Testing](dev/pytest.md) - [Pull Requests](dev/PR.md) - [Testing](dev/testing.md) - [Code Reviews](dev/code_review.md) diff --git a/docs/user/fan.md b/docs/user/fan.md index 1392e24f4..4b8eddf3b 100644 --- a/docs/user/fan.md +++ b/docs/user/fan.md @@ -1,6 +1,7 @@