Simplify broadcastable a bit further and add docs (#26601)

* Make LinAlg fess up to using Base._return_type

* Simplify broadcastable a bit further and add docs

Amazing what a great exercise writing documentation is. This is a followup to #26435, and more clearly lays out the requirements for defining a custom (non-AbstractArray) type that can participate in broadcasting.  The biggest functional change here is that types that define their own BroadcastStyle must now also implement `broadcastable` to be a no-op.

* Trailing comma, doc fixes from review; more tests

base/broadcast.jl

@@ -7,7 +7,7 @@ using .Base: Indices, OneTo, linearindices, tail, to_shape,
             _msk_end, unsafe_bitgetindex, bitcache_chunks, bitcache_size, dumpbitcache,
             isoperator, promote_typejoin, unalias
 import .Base: broadcast, broadcast!
-export BroadcastStyle, broadcast_indices, broadcast_similar,
+export BroadcastStyle, broadcast_indices, broadcast_similar, broadcastable,
        broadcast_getindex, broadcast_setindex!, dotview, @__dot__
 
 ### Objects with customized broadcasting behavior should declare a BroadcastStyle
@@ -48,7 +48,6 @@ BroadcastStyle(::Type{<:Tuple}) = Style{Tuple}()
 
 struct Unknown <: BroadcastStyle end
 BroadcastStyle(::Type{Union{}}) = Unknown()  # ambiguity resolution
-BroadcastStyle(::Type) = Unknown()
 
 """
 `Broadcast.AbstractArrayStyle{N} <: BroadcastStyle` is the abstract supertype for any style
@@ -101,7 +100,8 @@ struct DefaultArrayStyle{N} <: AbstractArrayStyle{N} end
 const DefaultVectorStyle = DefaultArrayStyle{1}
 const DefaultMatrixStyle = DefaultArrayStyle{2}
 BroadcastStyle(::Type{<:AbstractArray{T,N}}) where {T,N} = DefaultArrayStyle{N}()
-BroadcastStyle(::Type{<:Union{Ref,Number}}) = DefaultArrayStyle{0}()
+BroadcastStyle(::Type{<:Ref}) = DefaultArrayStyle{0}()
+BroadcastStyle(::Type{T}) where {T} = DefaultArrayStyle{ndims(T)}()
 
 # `ArrayConflict` is an internal type signaling that two or more different `AbstractArrayStyle`
 # objects were supplied as arguments, and that no rule was defined for resolving the
@@ -385,9 +385,15 @@ end
 """
     broadcastable(x)
 
-Return either `x` or an object like `x` such that it supports `axes` and indexing.
+Return either `x` or an object like `x` such that it supports `axes`, indexing, and its type supports `ndims`.
 
-If `x` supports iteration, the returned value should have the same `axes` and indexing behaviors as [`collect(x)`](@ref).
+If `x` supports iteration, the returned value should have the same `axes` and indexing
+behaviors as [`collect(x)`](@ref).
+
+If `x` is not an `AbstractArray` but it supports `axes`, indexing, and its type supports
+`ndims`, then `broadcastable(::typeof(x))` may be implemented to just return itself.
+Further, if `x` defines its own [`BroadcastStyle`](@ref), then it must define its
+`broadcastable` method to return itself for the custom style to have any effect.
 
 # Examples
 ```jldoctest
@@ -407,9 +413,9 @@ Base.RefValue{String}("hello")
 broadcastable(x::Union{Symbol,AbstractString,Function,UndefInitializer,Nothing,RoundingMode,Missing}) = Ref(x)
 broadcastable(x::Ptr) = Ref{Ptr}(x) # Cannot use Ref(::Ptr) until ambiguous deprecation goes through
 broadcastable(::Type{T}) where {T} = Ref{Type{T}}(T)
-broadcastable(x::AbstractArray) = x
+broadcastable(x::Union{AbstractArray,Number,Ref,Tuple}) = x
 # In the future, default to collecting arguments. TODO: uncomment once deprecations are removed
-# broadcastable(x) = BroadcastStyle(typeof(x)) isa Unknown ? collect(x) : x
+# broadcastable(x) = collect(x)
 # broadcastable(::Union{AbstractDict, NamedTuple}) = error("intentionally unimplemented to allow development in 1.x")
 
 """
@@ -590,11 +596,6 @@ julia> abs.((1, -2))
 julia> broadcast(+, 1.0, (0, -2.0))
 (1.0, -1.0)
 
-julia> broadcast(+, 1.0, (0, -2.0), Ref(1))
-2-element Array{Float64,1}:
- 2.0
- 0.0
-
 julia> (+).([[0,2], [1,3]], Ref{Vector{Int}}([1,-1]))
 2-element Array{Array{Int64,1},1}:
  [1, 1]

base/deprecated.jl

@@ -689,15 +689,11 @@ end
 
 # Broadcast no longer defaults to treating its arguments as scalar (#)
 @noinline function Broadcast.broadcastable(x)
-    if Base.Broadcast.BroadcastStyle(typeof(x)) isa Broadcast.Unknown
-        depwarn("""
-            broadcast will default to iterating over its arguments in the future. Wrap arguments of
-            type `x::$(typeof(x))` with `Ref(x)` to ensure they broadcast as "scalar" elements.
-            """, (:broadcast, :broadcast!))
-        return Ref{typeof(x)}(x)
-    else
-        return x
-    end
+    depwarn("""
+        broadcast will default to iterating over its arguments in the future. Wrap arguments of
+        type `x::$(typeof(x))` with `Ref(x)` to ensure they broadcast as "scalar" elements.
+        """, (:broadcast, :broadcast!))
+    return Ref{typeof(x)}(x)
 end
 @eval Base.Broadcast Base.@deprecate_binding Scalar DefaultArrayStyle{0} false
 # After deprecation is removed, enable the fallback broadcastable definitions in base/broadcast.jl

doc/src/base/arrays.md

@@ -70,10 +70,10 @@ For specializing broadcast on custom types, see
 Base.BroadcastStyle
 Base.broadcast_similar
 Base.broadcast_indices
-Base.Broadcast.Scalar
 Base.Broadcast.AbstractArrayStyle
 Base.Broadcast.ArrayStyle
 Base.Broadcast.DefaultArrayStyle
+Base.Broadcast.broadcastable
 ```
 
 ## Indexing and assignment

doc/src/manual/interfaces.md

@@ -443,7 +443,8 @@ V = view(A, [1,2,4], :)   # is not strided, as the spacing between rows is not f
 | `Base.broadcast_similar(f, ::DestStyle, ::Type{ElType}, inds, As...)` | Allocation of output container |
 | **Optional methods** | | |
 | `Base.BroadcastStyle(::Style1, ::Style2) = Style12()` | Precedence rules for mixing styles |
-| `Base.broadcast_indices(::StyleA, A)` | Declaration of the indices of `A` for broadcasting purposes (for AbstractArrays, defaults to `axes(A)`) |
+| `Base.broadcast_indices(::StyleA, A)` | Declaration of the indices of `A` for broadcasting purposes (defaults to [`axes(A)`](@ref)) |
+| `Base.broadcastable(x)` | Convert `x` to an object that has `axes` and supports indexing |
 | **Bypassing default machinery** | |
 | `broadcast(f, As...)` | Complete bypass of broadcasting machinery |
 | `broadcast(f, ::DestStyle, ::Nothing, ::Nothing, As...)` | Bypass after container type is computed |
@@ -452,21 +453,43 @@ V = view(A, [1,2,4], :)   # is not strided, as the spacing between rows is not f
 | `broadcast!(f, dest, ::BroadcastStyle, As...)` | Bypass in-place broadcast, specialization on `BroadcastStyle` |
 
 [Broadcasting](@ref) is triggered by an explicit call to `broadcast` or `broadcast!`, or implicitly by
-"dot" operations like `A .+ b`. Any `AbstractArray` type supports broadcasting,
-but the default result (output) type is `Array`. To specialize the result for specific input type(s),
-the main task is the allocation of an appropriate result object.
-(This is not an issue for `broadcast!`, where
-the result object is passed as an argument.) This process is split into two stages: computation
-of the behavior and type from the arguments ([`Base.BroadcastStyle`](@ref)), and allocation of the object
-given the resulting type with [`Base.broadcast_similar`](@ref).
-
-`Base.BroadcastStyle` is an abstract type from which all styles are
+"dot" operations like `A .+ b` or `f.(x, y)`. Any object that has [`axes`](@ref) and supports
+indexing can participate as an argument in broadcasting, and by default the result is stored
+in an `Array`. This basic framework is extensible in three major ways:
+
+* Ensuring that all arguments support broadcast
+* Selecting an appropriate output array for the given set of arguments
+* Selecting an efficient implementation for the given set of arguments
+
+Not all types support `axes` and indexing, but many are convenient to allow in broadcast.
+The [`Base.broadcastable`](@ref) function is called on each argument to broadcast, allowing
+it to return something different that supports `axes` and indexing if it does not. By
+default, this is the identity function for all `AbstractArray`s and `Number`s — they already
+support `axes` and indexing. For a handful of other types (including but not limited to
+types themselves, functions, special singletons like `missing` and `nothing`, and dates),
+`Base.broadcastable` returns the argument wrapped in a `Ref` to act as a 0-dimensional
+"scalar" for the purposes of broadcasting. Custom types can similarly specialize
+`Base.broadcastable` to define their shape, but they should follow the convention that
+`collect(Base.broadcastable(x)) == collect(x)`. A notable exception are `AbstractString`s;
+they are special-cased to behave as scalars for the purposes of broadcast even though they
+are iterable collections of their characters.
+
+The next two steps (selecting the output array and implementation) are dependent upon
+determining a single answer for a given set of arguments. Broadcast must take all the varied
+types of its arguments and collapse them down to just one output array and one
+implementation. Broadcast calls this single answer a "style." Every broadcastable object
+each has its own preferred style, and a promotion-like system is used to combine these
+styles into a single answer — the "destination style".
+
+### Broadcast Styles
+
+`Base.BroadcastStyle` is the abstract type from which all styles are
 derived. When used as a function it has two possible forms,
 unary (single-argument) and binary.
 The unary variant states that you intend to
 implement specific broadcasting behavior and/or output type,
-and do not wish to rely on the default fallback ([`Broadcast.Scalar`](@ref) or [`Broadcast.DefaultArrayStyle`](@ref)).
-To achieve this, you can define a custom `BroadcastStyle` for your object:
+and do not wish to rely on the default fallback ([`Broadcast.DefaultArrayStyle`](@ref)).
+To override these defaults, you can define a custom `BroadcastStyle` for your object:
 
 ```julia
 struct MyStyle <: Broadcast.BroadcastStyle end
@@ -486,6 +509,8 @@ When your broadcast operation involves several arguments, individual argument st
 combined to determine a single `DestStyle` that controls the type of the output container.
 For more detail, see [below](@ref writing-binary-broadcasting-rules).
 
+### Selecting an appropriate output array
+
 The actual allocation of the result array is handled by `Base.broadcast_similar`:
 
 ```julia
@@ -562,6 +587,8 @@ julia> a .+ [5,10]
  13  14
 ```
 
+### [Extending broadcast with custom implementations](@id extending-in-place-broadcast)
+
 Finally, it's worth noting that sometimes it's easier simply to bypass the machinery for
 computing result types and container sizes, and just do everything manually. For example,
 you can convert a `UnitRange{Int}` `r` to a `UnitRange{BigInt}` with `big.(r)`; the definition
@@ -580,7 +607,40 @@ the internal machinery to compute the container type, element type, and indices
 Broadcast.broadcast(::typeof(somefunction), ::MyStyle, ::Type{ElType}, inds, As...)
 ```
 
-### [Writing binary broadcasting rules](@id writing-binary-broadcasting-rules)
+Extending `broadcast!` (in-place broadcast) should be done with care, as it is easy to introduce
+ambiguities between packages. To avoid these ambiguities, we adhere to the following conventions.
+
+First, if you want to specialize on the destination type, say `DestType`, then you should
+define a method with the following signature:
+
+```julia
+broadcast!(f, dest::DestType, ::Nothing, As...)
+```
+
+Note that no bounds should be placed on the types of `f` and `As...`.
+
+Second, if specialized `broadcast!` behavior is desired depending on the input types,
+you should write [binary broadcasting rules](@ref writing-binary-broadcasting-rules) to
+determine a custom `BroadcastStyle` given the input types, say `MyBroadcastStyle`, and you should define a method with the following
+signature:
+
+```julia
+broadcast!(f, dest, ::MyBroadcastStyle, As...)
+```
+
+Note the lack of bounds on `f`, `dest`, and `As...`.
+
+Third, simultaneously specializing on both the type of `dest` and the `BroadcastStyle` is fine. In this case,
+it is also allowed to specialize on the types of the source arguments (`As...`). For example, these method signatures are OK:
+
+```julia
+broadcast!(f, dest::DestType, ::MyBroadcastStyle, As...)
+broadcast!(f, dest::DestType, ::MyBroadcastStyle, As::AbstractArray...)
+broadcast!(f, dest::DestType, ::Broadcast.DefaultArrayStyle{0}, As::Number...)
+```
+
+
+#### [Writing binary broadcasting rules](@id writing-binary-broadcasting-rules)
 
 The precedence rules are defined by binary `BroadcastStyle` calls:
 
@@ -592,10 +652,10 @@ where `Style12` is the `BroadcastStyle` you want to choose for outputs involving
 arguments of `Style1` and `Style2`. For example,
 
 ```julia
-Base.BroadcastStyle(::Broadcast.Style{Tuple}, ::Broadcast.Scalar) = Broadcast.Style{Tuple}()
+Base.BroadcastStyle(::Broadcast.Style{Tuple}, ::Broadcast.AbstractArrayStyle{0}) = Broadcast.Style{Tuple}()
 ```
 
-indicates that `Tuple` "wins" over scalars (the output container will be a tuple).
+indicates that `Tuple` "wins" over zero-dimensional arrays (the output container will be a tuple).
 It is worth noting that you do not need to (and should not) define both argument orders
 of this call; defining one is sufficient no matter what order the user supplies the arguments in.
 
@@ -643,37 +703,3 @@ yields another `SparseVecStyle`, that its combination with a 2-dimensional array
 yields a `SparseMatStyle`, and anything of higher dimensionality falls back to the dense arbitrary-dimensional framework.
 These rules allow broadcasting to keep the sparse representation for operations that result
 in one or two dimensional outputs, but produce an `Array` for any other dimensionality.
-
-### [Extending `broadcast!`](@id extending-in-place-broadcast)
-
-Extending `broadcast!` (in-place broadcast) should be done with care, as it is easy to introduce
-ambiguities between packages. To avoid these ambiguities, we adhere to the following conventions.
-
-First, if you want to specialize on the destination type, say `DestType`, then you should
-define a method with the following signature:
-
-```julia
-broadcast!(f, dest::DestType, ::Nothing, As...)
-```
-
-Note that no bounds should be placed on the types of `f` and `As...`.
-
-Second, if specialized `broadcast!` behavior is desired depending on the input types,
-you should write [binary broadcasting rules](@ref writing-binary-broadcasting-rules) to
-determine a custom `BroadcastStyle` given the input types, say `MyBroadcastStyle`, and you should define a method with the following
-signature:
-
-```julia
-broadcast!(f, dest, ::MyBroadcastStyle, As...)
-```
-
-Note the lack of bounds on `f`, `dest`, and `As...`.
-
-Third, simultaneously specializing on both the type of `dest` and the `BroadcastStyle` is fine. In this case,
-it is also allowed to specialize on the types of the source arguments (`As...`). For example, these method signatures are OK:
-
-```julia
-broadcast!(f, dest::DestType, ::MyBroadcastStyle, As...)
-broadcast!(f, dest::DestType, ::MyBroadcastStyle, As::AbstractArray...)
-broadcast!(f, dest::DestType, ::Broadcast.Scalar, As::Number...)
-```

stdlib/LinearAlgebra/src/uniformscaling.jl

@@ -102,7 +102,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular),
             ($op)(UL::$t2, J::UniformScaling) = ($t2)(($op)(UL.data, J))
 
             function ($op)(UL::$t1, J::UniformScaling)
-                ULnew = copy_oftype(UL.data, Broadcast.combine_eltypes($op, UL, J))
+                ULnew = copy_oftype(UL.data, Base._return_type($op, Tuple{eltype(UL), typeof(J.λ)}))
                 for i = 1:size(ULnew, 1)
                     ULnew[i,i] = ($op)(1, J.λ)
                 end
@@ -113,7 +113,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular),
 end
 
 function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular})
-    ULnew = similar(parent(UL), Broadcast.combine_eltypes(-, J, UL))
+    ULnew = similar(parent(UL), Base._return_type(-, Tuple{typeof(J.λ), eltype(UL)}))
     n = size(ULnew, 1)
     ULold = UL.data
     for j = 1:n
@@ -129,7 +129,7 @@ function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular})
     return UpperTriangular(ULnew)
 end
 function (-)(J::UniformScaling, UL::Union{LowerTriangular,UnitLowerTriangular})
-    ULnew = similar(parent(UL), Broadcast.combine_eltypes(-, J, UL))
+    ULnew = similar(parent(UL), Base._return_type(-, Tuple{typeof(J.λ), eltype(UL)}))
     n = size(ULnew, 1)
     ULold = UL.data
     for j = 1:n
@@ -147,7 +147,7 @@ end
 
 function (+)(A::AbstractMatrix, J::UniformScaling)
     n = checksquare(A)
-    B = similar(A, Broadcast.combine_eltypes(+, A, J))
+    B = similar(A, Base._return_type(+, Tuple{eltype(A), typeof(J.λ)}))
     copyto!(B,A)
     @inbounds for i = 1:n
         B[i,i] += J.λ
@@ -157,7 +157,7 @@ end
 
 function (-)(A::AbstractMatrix, J::UniformScaling)
     n = checksquare(A)
-    B = similar(A, Broadcast.combine_eltypes(-, A, J))
+    B = similar(A, Base._return_type(-, Tuple{eltype(A), typeof(J.λ)}))
     copyto!(B, A)
     @inbounds for i = 1:n
         B[i,i] -= J.λ
@@ -166,7 +166,7 @@ function (-)(A::AbstractMatrix, J::UniformScaling)
 end
 function (-)(J::UniformScaling, A::AbstractMatrix)
     n = checksquare(A)
-    B = convert(AbstractMatrix{Broadcast.combine_eltypes(-, J, A)}, -A)
+    B = convert(AbstractMatrix{Base._return_type(-, Tuple{typeof(J.λ), eltype(A)})}, -A)
     @inbounds for j = 1:n
         B[j,j] += J.λ
     end

test/broadcast.jl

@@ -545,7 +545,7 @@ end
 # Test that broadcast treats type arguments as scalars, i.e. containertype yields Any,
 # even for subtypes of abstract array. (https://github.com/JuliaStats/DataArrays.jl/issues/229)
 @testset "treat type arguments as scalars, DataArrays issue 229" begin
-    @test Broadcast.combine_styles(AbstractArray) == Broadcast.Unknown()
+    @test Broadcast.combine_styles(Broadcast.broadcastable(AbstractArray)) == Base.Broadcast.DefaultArrayStyle{0}()
     @test broadcast(==, [1], AbstractArray) == BitArray([false])
     @test broadcast(==, 1, AbstractArray) == false
 end
@@ -578,6 +578,25 @@ end
         [Set([1, 3]), Set([2, 3])])
 end
 
+# A bare bones custom type that supports broadcast
+struct Foo26601{T}
+    data::T
+end
+Base.axes(f::Foo26601) = axes(f.data)
+Base.getindex(f::Foo26601, i...) = getindex(f.data, i...)
+Base.ndims(::Type{Foo26601{T}}) where {T} = ndims(T)
+Base.Broadcast.broadcastable(f::Foo26601) = f
+@testset "barebones custom object broadcasting" begin
+    for d in (rand(Float64, ()), rand(5), rand(5,5), rand(5,5,5))
+        f = Foo26601(d)
+        @test f .* 2 == d .* 2
+        @test f .* (1:5) == d .* (1:5)
+        @test f .* reshape(1:25,5,5) == d .* reshape(1:25,5,5)
+        @test sqrt.(f) == sqrt.(d)
+        @test f .* (1,2,3,4,5) == d .* (1,2,3,4,5)
+    end
+end
+
 @testset "broadcast resulting in tuples" begin
     # Issue #21291
     let t = (0, 1, 2)