limit maximum vector alignment to heap alignment

src/abi_arm.cpp

@@ -270,7 +270,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
     //   For a Composite Type, the alignment of the copy will have 4-byte
     //   alignment if its natural alignment is <= 4 and 8-byte alignment if
     //   its natural alignment is >= 8
-    size_t align = dt->layout->alignment;
+    size_t align = jl_datatype_align(dt);
     if (align < 4)
         align = 4;
     if (align > 8)

src/abi_ppc64le.cpp

@@ -144,7 +144,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
     // rewrite integer-sized (non-HFA) struct to an array
     // the bitsize of the integer gives the desired alignment
     if (size > 8) {
-        if (dt->layout->alignment <= 8) {
+        if (jl_datatype_align(dt) <= 8) {
             return ArrayType::get(T_int64, (size + 7) / 8);
         }
         else {

src/array.c

@@ -193,10 +193,10 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
     assert(store_unboxed(el_type) == !data->flags.ptrarray);
     if (!data->flags.ptrarray) {
         a->elsize = jl_datatype_size(el_type);
-        unsigned align = ((jl_datatype_t*)el_type)->layout->alignment;
+        unsigned align = jl_datatype_align(el_type);
         jl_value_t *ownerty = jl_typeof(owner);
         unsigned oldalign = (ownerty == (jl_value_t*)jl_string_type ? 1 :
-                             ((jl_datatype_t*)jl_tparam0(ownerty))->layout->alignment);
+                             jl_datatype_align(jl_tparam0(ownerty)));
         if (oldalign < align)
             jl_exceptionf(jl_argumenterror_type,
                           "reinterpret from alignment %u bytes to alignment %u bytes not allowed",
@@ -283,7 +283,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
     unsigned align;
     if (isunboxed) {
         elsz = jl_datatype_size(el_type);
-        align = ((jl_datatype_t*)el_type)->layout->alignment;
+        align = jl_datatype_align(el_type);
     }
     else {
         align = elsz = sizeof(void*);
@@ -346,7 +346,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
     unsigned align;
     if (isunboxed) {
         elsz = jl_datatype_size(el_type);
-        align = ((jl_datatype_t*)el_type)->layout->alignment;
+        align = jl_datatype_align(el_type);
     }
     else {
         align = elsz = sizeof(void*);

src/ccall.cpp

@@ -618,7 +618,7 @@ static Value *julia_to_address(Type *to, jl_value_t *jlto, jl_unionall_t *jlto_e
         builder.CreateMemCpy(slot,
                              data_pointer(jvinfo, ctx, slot->getType()),
                              (uint64_t)jl_datatype_size(ety),
-                             (uint64_t)((jl_datatype_t*)ety)->layout->alignment);
+                             (uint64_t)jl_datatype_align(ety));
         mark_gc_use(jvinfo);
     }
     if (slot->getType() != to)
@@ -657,7 +657,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, jl_union
         builder.CreateMemCpy(slot,
                              data_pointer(jvinfo, ctx, slot->getType()),
                              (uint64_t)jl_datatype_size(jlto),
-                             (uint64_t)((jl_datatype_t*)jlto)->layout->alignment);
+                             (uint64_t)jl_datatype_align(jlto));
         mark_gc_use(jvinfo);
     }
     return slot;

src/cgutils.cpp

@@ -157,15 +157,15 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed
         llvm::DIType *t = dbuilder->createBasicType(
                 jl_symbol_name(jdt->name->name),
                 SizeInBits,
-                8 * jdt->layout->alignment,
+                8 * jl_datatype_align(jdt),
                 llvm::dwarf::DW_ATE_unsigned);
         jdt->ditype = t;
         return t;
 #else
         DIType t = dbuilder->createBasicType(
                 jl_symbol_name(jdt->name->name),
                 SizeInBits,
-                8 * jdt->layout->alignment,
+                8 * jl_datatype_align(jdt),
                 llvm::dwarf::DW_ATE_unsigned);
         MDNode *M = t;
         jdt->ditype = M;
@@ -189,7 +189,7 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed
             NULL,                       // File
             0,                          // LineNumber
             jl_datatype_nbits(jdt),     // SizeInBits
-            8 * jdt->layout->alignment, // AlignInBits
+            8 * jl_datatype_align(jdt), // AlignInBits
             DIFlagZero,                 // Flags
             NULL,                       // DerivedFrom
             DINodeArray(),              // Elements
@@ -523,8 +523,12 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox
                     *jl_ExecutionEngine->getDataLayout();
 #endif
                 unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl);
-                unsigned julia_alignment = jst->layout->alignment;
-                assert(llvm_alignment == julia_alignment);
+                unsigned julia_alignment = jl_datatype_align(jst);
+                // Check that the alignment adheres to the heap alignment.
+                assert(julia_alignment <= JL_HEAP_ALIGNMENT);
+                // TODO: Fix alignment calculation in LLVM, as well as in the GC and the struct declaration
+                if (llvm_alignment  <= JL_HEAP_ALIGNMENT)
+                    assert(julia_alignment == llvm_alignment);
             }
 #endif
         }
@@ -1165,9 +1169,10 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value *
 // It is currently unused, but might be used in the future for a more precise answer.
 static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment)
 {
-    if (!alignment && ((jl_datatype_t*)jltype)->layout->alignment > MAX_ALIGN) {
-        // Type's natural alignment exceeds strictest alignment promised in heap, so return the heap alignment.
-        return MAX_ALIGN;
+    if (!alignment) {
+        alignment = jl_datatype_align(jltype);
+        assert(alignment <= JL_HEAP_ALIGNMENT);
+        assert(JL_HEAP_ALIGNMENT % alignment == 0);
     }
     return alignment;
 }

src/codegen.cpp

@@ -3688,7 +3688,7 @@ static void union_alloca_type(jl_uniontype_t *ut,
             [&](unsigned idx, jl_datatype_t *jt) {
                 if (!jl_is_datatype_singleton(jt)) {
                     size_t nb1 = jl_datatype_size(jt);
-                    size_t align1 = jt->layout->alignment;
+                    size_t align1 = jl_datatype_align(jt);
                     if (nb1 > nbytes)
                         nbytes = nb1;
                     if (align1 > align)
@@ -3959,7 +3959,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx)
                     builder.CreateMemCpy(vi.value.V,
                                          data_pointer(rval_info, ctx, T_pint8),
                                          copy_bytes,
-                                         ((jl_datatype_t*)rval_info.typ)->layout->alignment,
+                                         jl_datatype_align(rval_info.typ),
                                          vi.isVolatile,
                                          tbaa);
                 }

src/datatype.c

@@ -278,7 +278,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
             // Should never happen
             if (__unlikely(fsz > max_size))
                 goto throw_ovf;
-            al = ((jl_datatype_t*)ty)->layout->alignment;
+            al = jl_datatype_align(ty);
             desc[i].isptr = 0;
             if (((jl_datatype_t*)ty)->layout->haspadding)
                 haspadding = 1;
@@ -290,6 +290,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
             al = fsz;
             desc[i].isptr = 1;
         }
+        assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0);
         if (al != 0) {
             size_t alsz = LLT_ALIGN(sz, al);
             if (sz & (al - 1))
@@ -310,7 +311,10 @@ void jl_compute_field_offsets(jl_datatype_t *st)
         // Some tuples become LLVM vectors with stronger alignment than what was calculated above.
         unsigned al = jl_special_vector_alignment(nfields, lastty);
         assert(al % alignm == 0);
-        if (al)
+        // JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap.
+        if (al > JL_HEAP_ALIGNMENT)
+            alignm = JL_HEAP_ALIGNMENT;
+        else if (al)
             alignm = al;
     }
     st->size = LLT_ALIGN(sz, alignm);
@@ -438,7 +442,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len)
     size_t nb = jl_datatype_size(bt);
     if (nb == 0)
         return jl_new_struct_uninit(bt);
-    *len = LLT_ALIGN(*len, bt->layout->alignment);
+    *len = LLT_ALIGN(*len, jl_datatype_align(bt));
     data = (char*)data + (*len);
     *len += nb;
     if (bt == jl_uint8_type)   return jl_box_uint8(*(uint8_t*)data);
@@ -759,7 +763,7 @@ JL_DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty)
 {
     if (ty->layout == NULL)
         jl_error("non-leaf type doesn't have an alignment");
-    return ty->layout->alignment;
+    return jl_datatype_align(ty);
 }
 
 #ifdef __cplusplus

src/gc.c

@@ -706,6 +706,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)
 {
     maybe_collect(ptls);
     size_t offs = offsetof(bigval_t, header);
+    static_assert(sizeof(bigval_t) % JL_HEAP_ALIGNMENT == 0, "");
     size_t allocsz = LLT_ALIGN(sz + offs, JL_CACHE_BYTE_ALIGNMENT);
     if (allocsz < sz)  // overflow in adding offs, size was "negative"
         jl_throw(jl_memory_exception);

src/gc.h

@@ -32,7 +32,7 @@ extern "C" {
 
 #define GC_PAGE_LG2 14 // log2(size of a page)
 #define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k
-#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT))
+#define GC_PAGE_OFFSET (JL_HEAP_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_HEAP_ALIGNMENT))
 
 #define jl_malloc_tag ((void*)0xdeadaa01)
 #define jl_singleton_tag ((void*)0xdeadaa02)

src/intrinsics.cpp

@@ -331,7 +331,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt, Value *d
 
     int alignment;
     if (x.isboxed) {
-         // julia's gc gives 16-byte aligned addresses
+        // julia's gc gives 16-byte aligned addresses
         alignment = 16;
     }
     else if (jt) {
@@ -616,7 +616,7 @@ static jl_cgval_t emit_pointerref(jl_cgval_t *argv, jl_codectx_t *ctx)
         Value *strct = emit_allocobj(ctx, size,
                                      literal_pointer_val((jl_value_t*)ety));
         im1 = builder.CreateMul(im1, ConstantInt::get(T_size,
-                    LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment)));
+                    LLT_ALIGN(size, jl_datatype_align(ety))));
         Value *thePtr = emit_unbox(T_pint8, e, e.typ);
         thePtr = builder.CreateGEP(emit_bitcast(thePtr, T_pint8), im1);
         builder.CreateMemCpy(emit_bitcast(strct, T_pint8), thePtr, size, 1);
@@ -673,7 +673,7 @@ static jl_cgval_t emit_pointerset(jl_cgval_t *argv, jl_codectx_t *ctx)
         thePtr = emit_unbox(T_pint8, e, e.typ);
         uint64_t size = jl_datatype_size(ety);
         im1 = builder.CreateMul(im1, ConstantInt::get(T_size,
-                    LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment)));
+                    LLT_ALIGN(size, jl_datatype_align(ety))));
         builder.CreateMemCpy(builder.CreateGEP(thePtr, im1),
                              data_pointer(x, ctx, T_pint8), size, align_nb);
     }

src/julia.h

@@ -773,6 +773,7 @@ STATIC_INLINE void jl_array_uint8_set(void *a, size_t i, uint8_t x)
 #define jl_field_type(st,i)    jl_svecref(((jl_datatype_t*)st)->types, (i))
 #define jl_field_count(st)     jl_svec_len(((jl_datatype_t*)st)->types)
 #define jl_datatype_size(t)    (((jl_datatype_t*)t)->size)
+#define jl_datatype_align(t)   (((jl_datatype_t*)t)->layout->alignment)
 #define jl_datatype_nbits(t)   ((((jl_datatype_t*)t)->size)*8)
 #define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->layout->nfields)
 

src/julia_internal.h

@@ -223,6 +223,8 @@ STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz)
 #endif
 #define JL_SMALL_BYTE_ALIGNMENT 16
 #define JL_CACHE_BYTE_ALIGNMENT 64
+// JL_HEAP_ALIGNMENT is the maximum alignment that the GC can provide
+#define JL_HEAP_ALIGNMENT JL_SMALL_BYTE_ALIGNMENT
 #define GC_MAX_SZCLASS (2032-sizeof(void*))
 
 STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)

src/runtime_intrinsics.c

@@ -46,7 +46,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t
     else {
         if (!jl_is_datatype(ety))
             jl_error("pointerref: invalid pointer");
-        size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment);
+        size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety));
         char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
         return jl_new_bits(ety, pp);
     }
@@ -67,7 +67,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t
     else {
         if (!jl_is_datatype(ety))
             jl_error("pointerset: invalid pointer");
-        size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment);
+        size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety));
         char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
         if (jl_typeof(x) != ety)
             jl_error("pointerset: type mismatch in assign");

test/vecelement.jl

@@ -23,6 +23,13 @@ for i=1:20
     end
 end
 
+# Try various large tuple lengths and element types #20961
+for i in (34, 36, 48, 64, 72, 80, 96)
+    for t in [Bool, Int8, Int16, Int32, Int64, Float32, Float64]
+        call_iota(i,t)
+    end
+end
+
 # Another crash report for #15244 motivated this test.
 struct Bunch{N,T}
     elts::NTuple{N,Base.VecElement{T}}
@@ -65,3 +72,50 @@ a[1] = Gr(5.0, Bunch((VecElement(6.0), VecElement(7.0))), 8.0)
 @test a[2] == Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0)
 
 @test isa(VecElement((1,2)), VecElement{Tuple{Int,Int}})
+
+# The following test mimic SIMD.jl
+const _llvmtypes = Dict{DataType, String}(
+    Float64 => "double",
+    Float32 => "float",
+    Int32 => "i32",
+    Int64 => "i64"
+)
+
+@generated function vecadd(x::Vec{N, T}, y::Vec{N, T}) where {N, T}
+    llvmT = _llvmtypes[T]
+    func = T <: AbstractFloat ? "fadd" : "add"
+    exp = """
+    %3 = $(func) <$(N) x $(llvmT)> %0, %1
+    ret <$(N) x $(llvmT)> %3
+    """
+    return quote
+        Base.@_inline_meta
+        Base.llvmcall($exp, Vec{$N, $T}, Tuple{Vec{$N, $T}, Vec{$N, $T}}, x, y)
+    end
+end
+
+function f20961(x::Vector{Vec{N, T}}, y::Vector{Vec{N, T}}) where{N, T}
+    @inbounds begin
+        a = x[1]
+        b = y[1]
+        return vecadd(a, b)
+    end
+end
+
+# Test various SIMD Vectors with known good sizes
+for T in (Float64, Float32, Int64, Int32)
+    for N in 1:36
+        # For some vectortypes Julia emits llvm arrays instead of vectors
+        if N %  7 == 0 || N % 11 == 0 || N % 13 == 0 || N % 15 == 0 ||
+           N % 19 == 0 || N % 23 == 0 || N % 25 == 0 || N % 27 == 0 ||
+           N % 29 == 0 || N % 31 == 0
+            continue
+        end
+        a = ntuple(i->VecElement(T(i)), N)
+        result = ntuple(i-> VecElement(T(i+i)), N)
+        b = vecadd(a, a)
+        @test b == result
+        b = f20961([a], [a])
+        @test b == result
+    end
+end