conda-build Mind Map

[kenodegard]

What is the idea?

Caution

This is a WIP!!

This diagream doesn't necessarily showcase the state of the world and is rather a mix of what conda-build does today and what we'd like it to do (e.g., conda-build isn't currently pluggable but the diagram below indicates several locations for plugins).

Please join the conversation below.

This is a diagram describing the process of taking a recipe and producing installable packages. The objective here is to:

1. define clear terminology to use for the different parts of conda-build and
2. create a mind map for conda-build so we may better see what functionality is more closely related to others so we may begin to reorient the code base accordingly.

Once we are happy with this diagram we should aspire to put it someplace more permanent.

%%{
  init: {
    'theme': 'base',
    'themeVariables': {
      'lineColor': '#00818C'
    }
  }
}%%
flowchart TB
    classDef input fill:#93C5FD,stroke:#3B82F6;
    classDef event fill:#fff,stroke:#333;
    classDef stage fill:#fff,stroke:#333;
    classDef outputs fill:#F0FDFA,stroke:#CCFBF1;
    classDef output fill:#99F6E4,stroke:#2DD4BF;
    classDef post fill:#F3E8FF,stroke:#E9D5FF;
    classDef warning fill:#FEF9E7,stroke:#FEF9E7;
    classDef plugin fill:#D8B4FE,stroke:#9333EA;
    classDef fail fill:#FEF2F2,stroke:#FECACA;
    classDef error fill:#FCA5A5,stroke:#EF4444;

    recipe:::stage
    subgraph recipe
        cbc_yml:::input
        cbc_yml("📄 conda_build_config.yml")
        
        meta_yml:::input
        meta_yml("📄 (template) meta.yml")

        args:::stage
        subgraph args
            append_file_cli:::input
            append_file_cli("📄 --append-file")

            clobber_file_cli:::input
            clobber_file_cli("📄 --clobber-file")

            variants_cli:::input
            variants_cli("📄 --variants")
        end
    end
    
    scripts:::stage
    subgraph scripts
        buildsh:::input
        buildsh("📄 build.sh")

        buildbat:::input
        buildbat("📄 build.bat")
    end

    tests:::stage
    subgraph tests
        runtestsh:::input
        runtestsh("📄 run_test.sh")

        runtestbat:::input
        runtestbat("📄 run_test.bat")

        runtestpy:::input
        runtestpy("📄 run_test.py")

        runtestpl:::input
        runtestpl("📄 run_test.pl")
    end

    context:::stage
    subgraph context ["context/Config"]
        cli:::input
        cli("</> CLI args")

        envvar:::input
        envvar("</> Environment variables")

        condarc:::input
        condarc("📄 condarc")
    end

    recipe --> render_stage
    scripts --> compile_stage
    tests --> test_stage
    context --> render_stage & compile_stage & package_stage

    workflow:::stage
    subgraph workflow
        render:::stage
        subgraph render
            render_stage:::stage
            subgraph render_stage ["[stage] render"]
                render_stage_note:::warning
                render_stage_note(["⚠️ Expands variants & multi-outputs"])

                render_stage1:::event
                render_stage1("Render Recipe(s)")
            end

            render_outputs:::outputs
            subgraph render_outputs ["[output] render"]
                render_output1:::output
                render_output1("📄 (rendered) meta.yml")
                
                render_output2:::output
                render_output2("📄 (rendered) meta.yml")
                
                render_output3:::output
                render_output3("📄")
            end

            render_post:::post
            subgraph render_post ["[post] render"]
                render_post1:::plugin
                render_post1("🧩 Dependency check (solve for test env)")
                
                render_post2:::plugin
                render_post2("🧩 Dependency check (solve for test env)")
                
                render_post3:::plugin
                render_post3("🧩")
            end
        end

        envs:::stage
        subgraph envs
            envs_stage:::stage
            subgraph envs_stage ["[stage] environments"]
                envs_stage1:::event
                envs_stage1("Create host & build envs")
                
                envs_stage2:::event
                envs_stage2("Create host & build envs")
                
                envs_stage3:::event
                envs_stage3("...")
            end

            envs_post:::post
            subgraph envs_post ["[post] environments"]
                envs_post1:::plugin
                envs_post1("🧩")
                
                envs_post2:::plugin
                envs_post2("🧩")
                
                envs_post3:::plugin
                envs_post3("🧩")
            end
        end

        fetch:::stage
        subgraph fetch
            fetch_stage:::stage
            subgraph fetch_stage ["[stage] fetch"]
                fetch_stage1:::event
                fetch_stage1("Git clone/local copy")
                
                fetch_stage2:::event
                fetch_stage2("Git clone/local copy")
                
                fetch_stage3:::event
                fetch_stage3("...")
            end

            fetch_post:::post
            subgraph fetch_post ["[post] fetch"]
                fetch_post1:::plugin
                fetch_post1("🧩 Apply patches")
                
                fetch_post2:::plugin
                fetch_post2("🧩 Apply patches")
                
                fetch_post3:::plugin
                fetch_post3("🧩")
            end
        end

        compile:::stage
        subgraph compile
            compile_stage:::stage
            subgraph compile_stage ["[stage] compile"]
                compile_stage1:::event
                compile_stage1("Construct/build/compile")
                
                compile_stage2:::event
                compile_stage2("Construct/build/compile")
                
                compile_stage3:::event
                compile_stage3("...")
            end

            compile_post:::post
            subgraph compile_post ["[post] compile"]
                compile_post_note:::warning
                compile_post_note(["⚠️ conda-verify?"])

                compile_post1:::plugin
                compile_post1("🧩 Overlinking & overdepending")
                
                compile_post2:::plugin
                compile_post2("🧩 Overlinking & overdepending")
                
                compile_post3:::plugin
                compile_post3("🧩")
            end
        end

        package:::stage
        subgraph package
            package_stage:::stage
            subgraph package_stage ["[stage] package"]
                package_stage1:::event
                package_stage1("Bundle")
                
                package_stage2:::event
                package_stage2("Bundle")
                
                package_stage3:::event
                package_stage3("...")
            end
            
            package_outputs:::outputs
            subgraph package_outputs ["[outputs] package"]
                package_output1:::output
                package_output1("📄 .tar.bz/.conda")
                
                package_output2:::output
                package_output2("📄 .tar.bz/.conda")
                
                package_output3:::output
                package_output3("📄")
            end
            
            package_post:::post
            subgraph package_post ["[post] package"]
                package_post1:::plugin
                package_post1("🧩")
                
                package_post2:::plugin
                package_post2("🧩")
                
                package_post3:::plugin
                package_post3("🧩")
            end
        end

        index:::stage
        subgraph index
            index_stage:::stage
            subgraph index_stage ["[stage] index"]
                index_stage1:::event
                index_stage1("Re-index local channel")
                
                index_stage2:::event
                index_stage2("Re-index local channel")
                
                index_stage3:::event
                index_stage3("...")
            end
            
            index_outputs:::outputs
            subgraph index_outputs ["[outputs] index"]
                index_output1:::output
                index_output1("📄 repodata.json")
                
                index_output2:::output
                index_output2("📄 repodata.json")
                
                index_output3:::output
                index_output3("📄")
            end

            index_post:::post
            subgraph index_post ["[post] index"]
                index_post1:::plugin
                index_post1("🧩")
                
                index_post2:::plugin
                index_post2("🧩")
                
                index_post3:::plugin
                index_post3("🧩")
            end
        end

        test:::stage
        subgraph test
            test_stage:::stage
            subgraph test_stage ["[stage] test"]
                test_stage1:::event
                test_stage1("Install & test package in test env")
                
                test_stage2:::event
                test_stage2("Install & test package in test env")
                
                test_stage3:::event
                test_stage3("...")
            end
            
            test_post:::post
            subgraph test_post ["[post] test"]
                test_post1:::plugin
                test_post1("🧩")
                
                test_post2:::plugin
                test_post2("🧩")
                
                test_post3:::plugin
                test_post3("🧩")
            end
        end

        cleanup:::stage
        subgraph cleanup
            cleanup_post:::post
            subgraph cleanup_post ["[post] cleanup"]
                cleanup_post1:::plugin
                cleanup_post1("🧩 Upload to ananconda.org")
                
                cleanup_post2:::plugin
                cleanup_post2("🧩 Upload to ananconda.org")
                
                cleanup_post3:::plugin
                cleanup_post3("🧩")
            end
        end

        fail:::stage
        subgraph fail
            render_fail:::fail
            subgraph render_fail ["[fail] render"]
                render_fail1:::error
                render_fail1("🧩")
            end

            envs_fail:::fail
            subgraph envs_fail ["[fail] envs"]
                envs_fail1:::error
                envs_fail1("🧩")
            end

            fetch_fail:::fail
            subgraph fetch_fail ["[fail] fetch"]
                fetch_fail1:::error
                fetch_fail1("🧩")
            end

            compile_fail:::fail
            subgraph compile_fail ["[fail] compile"]
                compile_fail1:::error
                compile_fail1("🧩")
            end

            package_fail:::fail
            subgraph package_fail ["[fail] package"]
                package_fail1:::error
                package_fail1("🧩")
            end

            index_fail:::fail
            subgraph index_fail ["[fail] index"]
                index_fail1:::error
                index_fail1("🧩")
            end

            test_fail:::fail
            subgraph test_fail ["[fail] test"]
                test_fail1:::error
                test_fail1("🧩")
            end

            cleanup_fail:::fail
            subgraph cleanup_fail ["[fail] cleanup"]
                cleanup_fail1:::error
                cleanup_fail1("🧩 Move to broken")
            end
        end
    end

    render_stage1 --> render_output1 --> render_post1 --> envs_stage1
    render_stage1 --> render_output2 --> render_post2 --> envs_stage2
    render_stage1 -.-> render_output3 -.-> render_post3 -.-> envs_stage3
    
    envs_stage1 --> envs_post1 --> fetch_stage1
    envs_stage2 --> envs_post2 --> fetch_stage2
    envs_stage3 -.-> envs_post3 -.-> fetch_stage3

    fetch_stage1 --> fetch_post1 --> compile_stage1
    fetch_stage2 --> fetch_post2 --> compile_stage2
    fetch_stage3 -.-> fetch_post3 -.-> compile_stage3

    compile_stage1 --> compile_post1 --> package_stage1
    compile_stage2 --> compile_post2 --> package_stage2
    compile_stage3 -.-> compile_post3 -.-> package_stage3

    package_stage1 --> package_output1 --> package_post1 --> index_stage1
    package_stage2 --> package_output2 --> package_post2 --> index_stage2
    package_stage3 -.-> package_output3 -.-> package_post3 -.-> index_stage3

    index_stage1 --> index_output1 --> index_post1 --> test_stage1
    index_stage2 --> index_output2 --> index_post2 --> test_stage2
    index_stage3 -.-> index_output3 -.-> index_post3 -.-> test_stage3

    test_stage1 --> test_post1 --> cleanup_post1
    test_stage2 --> test_post2 --> cleanup_post2
    test_stage3 -.-> test_post3 -.-> cleanup_post3

    render --> render_fail --> cleanup_fail
    envs --> envs_fail --> cleanup_fail
    fetch --> fetch_fail --> cleanup_fail
    compile --> compile_fail --> cleanup_fail
    package --> package_fail --> cleanup_fail
    index --> index_fail --> cleanup_fail
    test --> test_fail --> cleanup_fail
    cleanup --> cleanup_fail

[h-vetinari]

This is cool, though almost impossible to read in dark mode.

[jezdez]

https://github.com/orgs/community/discussions/35733 seems to indicate that you can move all graphs into a subgraph and it should at least render a background

[kenodegard]

@h-vetinari @jezdez should be fixed now