Reduce Sequali and (slightly) FastQC memory footprint (#2516) #2349
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "MultiQC" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - "docs/**" | |
| - "*.md" | |
| pull_request: | |
| paths-ignore: | |
| - "docs/**" | |
| - "*.md" | |
| env: | |
| # No need to run test variations across all Python versions | |
| latest_python: "3.12" | |
| jobs: | |
| changes: | |
| name: Check for changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| single_module: ${{ steps.single_module.outputs.value }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| modules: | |
| - "multiqc/modules/**" | |
| base_module: | |
| - "multiqc/modules/base_module.py" | |
| changelog: | |
| - "CHANGELOG.md" | |
| all_files: | |
| - "**" | |
| # Set the `-m` flag if only one module has changed | |
| - name: "Get a `-m` flag for the `multiqc` if only one module has changed" | |
| uses: haya14busa/action-cond@v1 | |
| id: single_module | |
| with: | |
| cond: ${{ ! steps.filter.outputs.changes.base_module && steps.filter.outputs.changes.n_modules == 1 && (steps.filter.outputs.changes.n_files_changes == 1 || steps.filter.outputs.changes.n_files_changes == 2 && steps.filter.outputs.changes.changelog)}} | |
| if_true: "-m ${{ steps.filter.outputs.changes.modules[0] }}" | |
| if_false: "" | |
| run_linux: | |
| name: Linux - Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-20.04 | |
| needs: changes | |
| strategy: | |
| matrix: | |
| # Just test the oldest and newest supported Python versions | |
| python-version: ["3.8", "3.12"] | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| env: | |
| GH_TOKEN: ${{ secrets.MQC_BOT_GITHUB_TOKEN }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: yezz123/setup-uv@v4 | |
| - name: Set the VIRTUAL_ENV variable for uv to work | |
| run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV | |
| - name: Install MultiQC | |
| run: uv pip install . | |
| - name: Download test data | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: MultiQC/test-data | |
| path: test_data | |
| # Run all of the tests! | |
| - name: All modules / Custom report filename | |
| run: multiqc ${{ needs.changes.outputs.single_module }} --strict test_data/data/modules/ --filename full_report.html | |
| # More variations of command line flags and inputs. No need to run those across all Python versions. | |
| - name: Special case input data | |
| if: matrix.python-version == env.latest_python | |
| run: multiqc ${{ needs.changes.outputs.single_module }} test_data/data --ignore test_data/data/modules/ | |
| - name: All modules / Log filename as s_name, no cleaning | |
| if: matrix.python-version == env.latest_python | |
| run: multiqc ${{ needs.changes.outputs.single_module }} --strict test_data/data/modules/ --fullnames --fn_as_s_name | |
| - name: Filter out all filenames (confirm no report) | |
| if: matrix.python-version == env.latest_python | |
| run: | | |
| multiqc ${{ needs.changes.outputs.single_module }} test_data/data/modules/ --filename all_ignored.html --ignore-samples '*' | |
| if [[ -f all_ignored.html ]]; then | |
| echo "No report should have been generated! Likely due to a module missing self.ignore_samples() or ModuleNoSamplesFound when no matching files are found" >&2 | |
| exit 1 | |
| fi | |
| - name: File-list input of dirs | |
| if: matrix.python-version == env.latest_python | |
| run: | | |
| cd test_data | |
| multiqc ${{ needs.changes.outputs.single_module }} --file-list data/special_cases/dir_list.txt | |
| - name: Specific module / Force overwrite / Prepend dirnames / Name and comment / No data dir | |
| if: matrix.python-version == env.latest_python | |
| run: multiqc --strict test_data/data/modules/ -m fastqc -f -d -dd 1 -i "Forced Report" -b "This command has lots of options" --filename custom_fn --no-data-dir | |
| - name: For overwrite / Flat plots / Exclude module / Ignore samples / Full names / zip data dir / config file | |
| if: matrix.python-version == env.latest_python | |
| run: multiqc ${{ needs.changes.outputs.single_module }} --strict test_data/data/modules/ -f --flat --exclude clusterflow --ignore-samples ngi --fullnames --zip-data-dir -c test/config_example.yaml | |
| - name: Specific module / named output dir / JSON data / File list | |
| if: matrix.python-version == env.latest_python | |
| run: | | |
| cd test_data | |
| multiqc -m star -o tests/multiqc_report_dev -k json --file-list data/special_cases/file_list.txt | |
| - name: No report | |
| if: matrix.python-version == env.latest_python | |
| run: | | |
| cd test_data | |
| multiqc -m star data/modules/star --no-report -n no-report | |
| if [[ -f no-report.html ]]; then | |
| echo "Running with the --no-report flag should not produce an HTML" >&2 | |
| exit 1 | |
| fi | |
| - name: Empty directory (confirm no report) | |
| if: matrix.python-version == env.latest_python | |
| run: | | |
| mkdir empty_dir | |
| multiqc -n empty empty_dir ${{ needs.changes.outputs.single_module }} | |
| if [[ -f empty.html ]]; then | |
| echo "Empty directory should not produce a report" >&2 | |
| exit 1 | |
| fi | |
| run_windows: | |
| name: Windows | |
| runs-on: windows-latest | |
| needs: changes | |
| timeout-minutes: 10 | |
| steps: | |
| # Check out MultiQC code | |
| - uses: actions/checkout@v4 | |
| # Set up Windows with C++ for numpy | |
| - name: Visual Studio Command Prompt tool | |
| run: cmd "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install MultiQC | |
| run: pip install . | |
| # Fetch the MultiQC test data | |
| # NB: Download zip file instead of git clone, as Windows complains about reserved filenames and certain characters | |
| - name: Download test data | |
| run: | | |
| curl -fsSL https://github.com/MultiQC/test-data/archive/main.zip -o test_data.zip | |
| 7z x test_data.zip -y -o"test_data" | |
| dir test_data\test-data-main | |
| # Run all of the tests! Remember the BACKslash path separator! | |
| - name: All modules / Custom report filename | |
| run: | | |
| $single_module = "${{ needs.changes.outputs.single_module }}" | |
| # want to run on just one module to avoid the windows job to time out: | |
| if ($single_module) { | |
| multiqc $single_module --strict test_data\test-data-main\data\modules\ --filename full_report.html | |
| } | |
| if (-not $single_module) { | |
| multiqc -m samtools --strict test_data\test-data-main\data\modules\ --filename full_report.html | |
| } |