fix #47658, state stack overflow on unions containing typevars (#48221)

(cherry picked from commit 596ce65)

Author: JeffBezanson

src/subtype.c

@@ -1174,6 +1174,19 @@ static int subtype_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_stenv_t *e, in
     return ans;
 }
 
+static int equal_unions(jl_uniontype_t *x, jl_uniontype_t *y, jl_stenv_t *e)
+{
+    jl_value_t *saved=NULL; jl_savedenv_t se;
+    JL_GC_PUSH1(&saved);
+    save_env(e, &saved, &se);
+    int eq = forall_exists_equal(x->a, y->a, e) && forall_exists_equal(x->b, y->b, e);
+    if (!eq)
+        restore_env(e, saved, &se);
+    free_env(&se);
+    JL_GC_POP();
+    return eq;
+}
+
 // `param` means we are currently looking at a parameter of a type constructor
 // (as opposed to being outside any type constructor, or comparing variable bounds).
 // this is used to record the positions where type variables occur for the
@@ -1363,6 +1376,27 @@ static int forall_exists_equal(jl_value_t *x, jl_value_t *y, jl_stenv_t *e)
         (is_definite_length_tuple_type(x) && is_indefinite_length_tuple_type(y)))
         return 0;
 
+    if (jl_is_uniontype(x) && jl_is_uniontype(y)) {
+        // For 2 unions, try a more efficient greedy algorithm that compares the unions
+        // componentwise. If it returns `false`, we forget it and proceed with the usual
+        // algorithm. If it returns `true` we try returning `true`, but need to come back
+        // here to try the usual algorithm if subtyping later fails.
+        jl_unionstate_t *state = &e->Runions;
+        jl_saved_unionstate_t oldRunions; push_unionstate(&oldRunions, state);
+        if (state->depth >= state->used) {
+            statestack_set(state, state->used, 0);
+            state->used++;
+        }
+        int ui = statestack_get(state, state->depth);
+        state->depth++;
+        if (ui == 0) {
+            state->more = state->depth; // memorize that this was the deepest available choice
+            if (equal_unions((jl_uniontype_t*)x, (jl_uniontype_t*)y, e))
+                return 1;
+            pop_unionstate(state, &oldRunions);
+        }
+    }
+
     jl_saved_unionstate_t oldLunions; push_unionstate(&oldLunions, &e->Lunions);
     e->Lunions.used = 0;
     int sub;

test/subtype.jl

@@ -2241,6 +2241,16 @@ let S = Tuple{T2, V2} where {T2, N2, V2<:(Array{S2, N2} where {S2 <: T2})},
     @testintersect(S, T, !Union{})
 end
 
+@test only(intersection_env(Val{Union{Val{Val{T}} where {T},Int}}, Val{Union{T,Int}} where T)[2]) === Val{Val{T}} where {T}
+
+# issue 47654
+Vec47654{T} = Union{AbstractVector{T}, AbstractVector{Union{T,Nothing}}}
+struct Wrapper47654{T, V<:Vec47654{T}}
+    v::V
+end
+abstract type P47654{A} end
+@test Wrapper47654{P47654, Vector{Union{P47654,Nothing}}} <: Wrapper47654
+
 @testset "known subtype/intersect issue" begin
     #issue 45874
     # Causes a hang due to jl_critical_error calling back into malloc...
@@ -2278,7 +2288,7 @@ end
     @test_broken typeintersect(Tuple{Type{Tuple{T,Val{T}}}, Val{T}} where T, Tuple{Type{Tuple{Val{T},T}}, Val{T}} where T) <: Any
 
     # issue 24333
-    @test_broken (Type{Union{Ref,Cvoid}} <: Type{Union{T,Cvoid}} where T)
+    @test (Type{Union{Ref,Cvoid}} <: Type{Union{T,Cvoid}} where T)
 
     # issue 22123
     t1 = Ref{Ref{Ref{Union{Int64, T}}} where T}
@@ -2289,4 +2299,16 @@ end
     @test_broken (Tuple{T1,T1} where T1<:(Val{T2} where T2)) <: (Tuple{Val{S},Val{S}} where S)
 end
 
+# issue #47658
+let T = Ref{NTuple{8, Ref{Union{Int, P}}}} where P,
+    S = Ref{NTuple{8, Ref{Union{Int, P}}}} where P
+    # note T and S are identical but we need 2 copies to avoid being fooled by pointer equality
+    @test T <: Union{Int, S}
+end
+
+# try to fool a greedy algorithm that picks X=Int, Y=String here
+@test Tuple{Ref{Union{Int,String}}, Ref{Union{Int,String}}} <: Tuple{Ref{Union{X,Y}}, Ref{X}} where {X,Y}
+# this slightly more complex case has been broken since 1.0 (worked in 0.6)
+@test_broken Tuple{Ref{Union{Int,String,Missing}}, Ref{Union{Int,String}}} <: Tuple{Ref{Union{X,Y}}, Ref{X}} where {X,Y}
+
 @test !(Tuple{Any, Any, Any} <: Tuple{Any, Vararg{T}} where T)