From b9316bd9e845c2128e68d57074c86ab785ade35e Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 21 Oct 2022 15:53:52 +0200 Subject: [PATCH] REL: v0.11.1 (#1089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * avoid creating many Annotations objects in for loop (#1079) * ENH: Better error message (#1080) * ENH: Better error message * FIX: Beautiful infrastructure * FIX: Spacing * ENH: Improve error message (#1081) * ENH: Improve error message * FIX: Use working openneuro-py * FIX: Install * FIX: Name * ENH: Make suggestion (#1087) * MRG: Enforce specification of all annotation descriptions in event_id if event_id is passed to write_raw_bids() (#1086) * Write all annotations even if event_id was passed * Implement new logic * Update changelog * Fix tests * Style * REL: v0.11.1 Co-authored-by: Alexandre Gramfort Co-authored-by: Eric Larson Co-authored-by: Richard Höchenberger --- doc/whats_new.rst | 11 +++++ mne_bids/__init__.py | 2 +- mne_bids/path.py | 12 ++++-- mne_bids/read.py | 44 ++++++++++++++++--- mne_bids/tests/test_path.py | 7 +++ mne_bids/tests/test_read.py | 10 +++++ mne_bids/tests/test_write.py | 84 ++++++++++++++++++++++++++++++++---- mne_bids/write.py | 19 ++++---- 8 files changed, 160 insertions(+), 29 deletions(-) diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 24af76e35..58dd61da8 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -7,6 +7,17 @@ What's new? =========== +.. _changes_0_11_1: + +Version 0.11.1 (2022-10-21) +--------------------------- + +Version 0.11.1 is a patch release, all changes are listed below. +For more complete information on the 0.11 version, see changelog below. + +- Speed up :func:`mne_bids.read_raw_bids` when lots of events are present by `Alexandre Gramfort`_ (:gh:`1079`) +- When writing data containing :class:`mne.Annotations` **and** passing events to :func:`~mne_bids.write_raw_bids`, previously, annotations whose description did not appear in ``event_id`` were silently dropped. We now raise an exception and request users to specify mappings between descriptions and event codes in this case. It is still possible to omit ``event_id`` if no ``events`` are passed, by `Richard Höchenberger`_ (:gh:`1084`) + .. _changes_0_11: Version 0.11 (2022-10-08) diff --git a/mne_bids/__init__.py b/mne_bids/__init__.py index 4929067c2..912c91649 100644 --- a/mne_bids/__init__.py +++ b/mne_bids/__init__.py @@ -1,6 +1,6 @@ """MNE software for easily interacting with BIDS compatible datasets.""" -__version__ = '0.11' +__version__ = '0.11.1' from mne_bids import commands from mne_bids.report import make_report from mne_bids.path import (BIDSPath, get_datatypes, get_entity_vals, diff --git a/mne_bids/path.py b/mne_bids/path.py index ecc19084a..f0328b9f3 100644 --- a/mne_bids/path.py +++ b/mne_bids/path.py @@ -953,8 +953,11 @@ def find_empty_room(self, use_sidecar_only=False, verbose=None): ) er_bids_path = _find_matched_empty_room(self) - if er_bids_path is not None: - assert er_bids_path.fpath.exists() + if er_bids_path is not None and not er_bids_path.fpath.exists(): + raise FileNotFoundError( + f'Empty-room BIDS path resolved but not found:\n' + f'{er_bids_path}\n' + 'Check your BIDS dataset for completeness.') return er_bids_path @@ -1510,9 +1513,10 @@ def _find_matching_sidecar(bids_path, suffix=None, f'associated with {bids_path.basename}.') elif len(best_candidates) > 1: # More than one candidates were tied for best match - msg = (f'Expected to find a single {suffix} file ' + msg = (f'Expected to find a single {search_suffix} file ' f'associated with {bids_path.basename}, ' - f'but found {len(candidate_list)}: "{candidate_list}".') + f'but found {len(candidate_list)}:\n\n' + + "\n".join(candidate_list)) msg += f'\n\nThe search_str was "{search_str_complete}"' if on_error == 'raise': raise RuntimeError(msg) diff --git a/mne_bids/read.py b/mne_bids/read.py index 976102b27..80cf5fea4 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -12,6 +12,7 @@ import re from datetime import datetime, timezone from difflib import get_close_matches +import os import numpy as np import mne @@ -123,8 +124,29 @@ def _read_events(events, event_id, raw, bids_path=None): else: events = read_events(events).astype(int) + if raw.annotations: + if event_id is None: + logger.info( + 'The provided raw data contains annotations, but you did not ' + 'pass an "event_id" mapping from annotation descriptions to ' + 'event codes. We will generate arbitrary event codes. ' + 'To specify custom event codes, please pass "event_id".' + ) + else: + desc_without_id = sorted( + set(raw.annotations.description) - set(event_id.keys()) + ) + if desc_without_id: + raise ValueError( + f'The provided raw data contains annotations, but ' + f'"event_id" does not contain entries for all annotation ' + f'descriptions. The following entries are missing: ' + f'{", ".join(desc_without_id)}' + ) + + # If we have events, convert them to Annotations so they can be easily + # merged with existing Annotations. if events.size > 0: - # Only keep events for which we have an ID <> description mapping. ids_without_desc = set(events[:, 2]) - set(event_id.values()) if ids_without_desc: raise ValueError( @@ -133,9 +155,6 @@ def _read_events(events, event_id, raw, bids_path=None): f'Please add them to the event_id dictionary, or drop them ' f'from the events array.' ) - del ids_without_desc - mask = [e in list(event_id.values()) for e in events[:, 2]] - events = events[mask] # Append events to raw.annotations. All event onsets are relative to # measurement beginning. @@ -491,8 +510,9 @@ def _handle_events_reading(events_fname, raw): description=descriptions) raw.set_annotations(annot_from_events) - annot_idx_to_keep = [idx for idx, annot in enumerate(annot_from_raw) - if annot['description'] in ANNOTATIONS_TO_KEEP] + annot_idx_to_keep = [idx for idx, descr + in enumerate(annot_from_raw.description) + if descr in ANNOTATIONS_TO_KEEP] annot_to_keep = annot_from_raw[annot_idx_to_keep] if len(annot_to_keep): @@ -719,7 +739,17 @@ def read_raw_bids(bids_path, extra_params=None, verbose=None): break if not raw_path.exists(): - raise FileNotFoundError(f'File does not exist: {raw_path}') + options = os.listdir(bids_path.directory) + matches = get_close_matches(bids_path.basename, options) + msg = f'File does not exist:\n{raw_path}' + if matches: + msg += ( + '\nDid you mean one of:\n' + + '\n'.join(matches) + + '\ninstead of:\n' + + bids_path.basename + ) + raise FileNotFoundError(msg) if config_path is not None and not config_path.exists(): raise FileNotFoundError(f'config directory not found: {config_path}') diff --git a/mne_bids/tests/test_path.py b/mne_bids/tests/test_path.py index 429852358..63229753e 100644 --- a/mne_bids/tests/test_path.py +++ b/mne_bids/tests/test_path.py @@ -960,11 +960,18 @@ def test_find_empty_room(return_bids_test_dir, tmp_path): # Retrieve empty-room BIDSPath assert bids_path.find_empty_room() == er_associated_bids_path + assert bids_path.find_empty_room( + use_sidecar_only=True) == er_associated_bids_path # Should only work for MEG with pytest.raises(ValueError, match='only supported for MEG'): bids_path.copy().update(datatype='eeg').find_empty_room() + # Raises an error if the file is missing + os.remove(er_associated_bids_path.fpath) + with pytest.raises(FileNotFoundError, match='Empty-room BIDS .* not foun'): + bids_path.find_empty_room(use_sidecar_only=True) + # Don't create `AssociatedEmptyRoom` entry in sidecar – we should now # retrieve the empty-room recording closer in time write_raw_bids(raw, bids_path=bids_path, empty_room=None, overwrite=True) diff --git a/mne_bids/tests/test_read.py b/mne_bids/tests/test_read.py index 7c1680798..64edcc793 100644 --- a/mne_bids/tests/test_read.py +++ b/mne_bids/tests/test_read.py @@ -1297,6 +1297,7 @@ def test_channels_tsv_raw_mismatch(tmp_path): read_raw_bids(bids_path) +@testing.requires_testing_data def test_file_not_found(tmp_path): """Check behavior if the requested file cannot be found.""" # First a path with a filename extension. @@ -1313,6 +1314,15 @@ def test_file_not_found(tmp_path): with pytest.raises(FileNotFoundError, match='File does not exist'): read_raw_bids(bids_path=bp) + bp.update(extension='.fif') + _read_raw_fif(raw_fname, verbose=False).save(bp.fpath) + with pytest.warns(RuntimeWarning, match=r'channels\.tsv'): + read_raw_bids(bp) # smoke test + + bp.update(task=None) + with pytest.raises(FileNotFoundError, match='Did you mean'): + read_raw_bids(bp) + @requires_version('mne', '1.2') @pytest.mark.filterwarnings(warning_str['channel_unit_changed']) diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py index 3f9b884c5..f55647e40 100644 --- a/mne_bids/tests/test_write.py +++ b/mne_bids/tests/test_write.py @@ -1272,17 +1272,10 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate, tmp_path): match='Encountered data in "double" format'): bids_output_path = write_raw_bids(**kwargs) - event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, - 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} - with pytest.raises(ValueError, match='You passed events, but no event_id '): write_raw_bids(raw, bids_path, events=events) - with pytest.raises(ValueError, - match='You passed event_id, but no events'): - write_raw_bids(raw, bids_path, event_id=event_id) - # check events.tsv is written events_tsv_fname = bids_output_path.copy().update(suffix='events', extension='.tsv') @@ -2704,6 +2697,81 @@ def test_annotations(_bids_validate, bad_segments, tmp_path): _bids_validate(bids_root) +@pytest.mark.parametrize( + 'write_events', [True, False] # whether to pass "events" to write_raw_bids +) +@pytest.mark.filterwarnings(warning_str['channel_unit_changed']) +@testing.requires_testing_data +def test_annotations_and_events(_bids_validate, tmp_path, write_events): + """Test combined writing of Annotations and events.""" + bids_root = tmp_path / 'bids' + bids_path = _bids_path.copy().update(root=bids_root, datatype='meg') + raw_fname = data_path / 'MEG' / 'sample' / 'sample_audvis_trunc_raw.fif' + events_fname = ( + data_path / 'MEG' / 'sample' / 'sample_audvis_trunc_raw-eve.fif' + ) + events_tsv_fname = bids_path.copy().update( + suffix='events', + extension='.tsv', + ) + + events = mne.read_events(events_fname) + events = events[events[:, 2] != 0] # drop unknown "0" events + event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, + 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} + raw = _read_raw_fif(raw_fname) + annotations = mne.Annotations( + # Try to avoid rounding errors. + onset=( + 1 / raw.info['sfreq'] * 600, + 1 / raw.info['sfreq'] * 600, # intentional + 1 / raw.info['sfreq'] * 3000 + ), + duration=( + 1 / raw.info['sfreq'], + 1 / raw.info['sfreq'], + 1 / raw.info['sfreq'] * 200 + ), + description=('BAD_segment', 'EDGE_segment', 'custom'), + ) + raw.set_annotations(annotations) + + # Write annotations while passing event_id + # Should raise since annotations descriptions are missing from event_id + with pytest.raises(ValueError, match='The following entries are missing'): + write_raw_bids( + raw, + bids_path=bids_path, + event_id=event_id, + events=events if write_events else None, + ) + + # Passing a complete mapping should work + event_id_with_annots = event_id.copy() + event_id_with_annots.update({ + 'BAD_segment': 9999, + 'EDGE_segment': 10000, + 'custom': 2000 + }) + write_raw_bids( + raw, + bids_path=bids_path, + event_id=event_id_with_annots, + events=events if write_events else None, + ) + _bids_validate(bids_root) + + # Ensure all events + annotations were written + events_tsv = _from_tsv(events_tsv_fname) + + if write_events: + n_events_expected = len(events) + len(raw.annotations) + else: + n_events_expected = len(raw.annotations) + + assert len(events_tsv['trial_type']) == n_events_expected + + @pytest.mark.parametrize( 'drop_undescribed_events', [True, False] @@ -2711,7 +2779,7 @@ def test_annotations(_bids_validate, bad_segments, tmp_path): @pytest.mark.filterwarnings(warning_str['channel_unit_changed']) @testing.requires_testing_data def test_undescribed_events(_bids_validate, drop_undescribed_events, tmp_path): - """Test we're behaving correctly if event descriptions are missing.""" + """Test we're raising if event descriptions are missing.""" bids_root = tmp_path / 'bids1' bids_path = _bids_path.copy().update(root=bids_root, datatype='meg') raw_fname = op.join(data_path, 'MEG', 'sample', diff --git a/mne_bids/write.py b/mne_bids/write.py index 59d56299e..0b5e54b1a 100644 --- a/mne_bids/write.py +++ b/mne_bids/write.py @@ -1337,14 +1337,16 @@ def write_raw_bids( If an array, the MNE events array (shape: ``(n_events, 3)``). If a path or an array and ``raw.annotations`` exist, the union of ``events`` and ``raw.annotations`` will be written. - Corresponding descriptions for all event codes (listed in the third + Mappings from event names to event codes (listed in the third column of the MNE events array) must be specified via the ``event_id`` - parameter; otherwise, an exception is raised. + parameter; otherwise, an exception is raised. If + :class:`~mne.Annotations` are present, their descriptions must be + included in ``event_id`` as well. If ``None``, events will only be inferred from the raw object's :class:`~mne.Annotations`. .. note:: - If ``not None``, writes the union of ``events`` and + If specified, writes the union of ``events`` and ``raw.annotations``. If you wish to **only** write ``raw.annotations``, pass ``events=None``. If you want to **exclude** the events in ``raw.annotations`` from being written, @@ -1359,7 +1361,10 @@ def write_raw_bids( ``events``. The descriptions will be written to the ``trial_type`` column in ``*_events.tsv``. The dictionary keys correspond to the event description,s and the values to the event codes. You must specify a - description for all event codes appearing in ``events``. + description for all event codes appearing in ``events``. If your data + contains :class:`~mne.Annotations`, you can use this parameter to + assign event codes to each unique annotation description (mapping from + description to event code). anonymize : dict | None If `None` (default), no anonymization is performed. If a dictionary, data will be anonymized depending on the dictionary @@ -1575,11 +1580,7 @@ def write_raw_bids( if events is not None and event_id is None: raise ValueError('You passed events, but no event_id ' - 'dictionary. You need to pass both, or neither.') - - if event_id is not None and events is None: - raise ValueError('You passed event_id, but no events. ' - 'You need to pass both, or neither.') + 'dictionary.') _validate_type(item=empty_room, item_name='empty_room', types=(mne.io.BaseRaw, BIDSPath, None))