Do not emit recursion groups without GC enabled (#4738)

We emit nominal types as a single large recursion group, but this produces
invalid modules when --nominal or --hybrid was used without GC enabled. Fix the
bug by always emitting types as though they were structural (i.e. without
recursion groups) when GC is not enabled.

Fixes #4723.

Author: tlively

src/wasm/wasm-binary.cpp

@@ -223,7 +223,12 @@ void WasmBinaryWriter::writeTypes() {
   // the type section. With nominal typing there is always one group and with
   // equirecursive typing there is one group per type.
   size_t numGroups = 0;
-  switch (getTypeSystem()) {
+  // MVP types are structural and do not use recursion groups.
+  TypeSystem typeSystem = getTypeSystem();
+  if (!wasm->features.hasGC()) {
+    typeSystem = TypeSystem::Equirecursive;
+  }
+  switch (typeSystem) {
     case TypeSystem::Equirecursive:
       numGroups = indexedTypes.types.size();
       break;
@@ -242,7 +247,7 @@ void WasmBinaryWriter::writeTypes() {
   BYN_TRACE("== writeTypes\n");
   auto start = startSection(BinaryConsts::Section::Type);
   o << U32LEB(numGroups);
-  if (getTypeSystem() == TypeSystem::Nominal) {
+  if (typeSystem == TypeSystem::Nominal) {
     // The nominal recursion group contains every type.
     o << S32LEB(BinaryConsts::EncodedType::Rec)
       << U32LEB(indexedTypes.types.size());

test/lit/nominal-no-gc.wast

@@ -0,0 +1,20 @@
+;; Write the module with --nominal but without GC
+;; RUN: wasm-opt %s --nominal --disable-gc -g -o %t.wasm
+
+;; We should not get any recursion groups even though we used --nominal. We use
+;; --hybrid -all here to make sure that any rec groups from the binary will
+;; actually show up in the output and cause the test to fail.
+;; RUN: wasm-opt %t.wasm --hybrid -all -S -o - | filecheck %s
+
+;; Also check that we don't get a failure with the default configuration.
+;; RUN: wasm-opt %t.wasm
+
+;; CHECK-NOT: rec
+
+(module
+  (type $foo (func))
+  (type $bar (func))
+
+  (func $f1 (type $foo))
+  (func $f2 (type $bar))
+)