[fbsync] Revert "[proto] Enable GPU tests on prototype (#6665)" (#6809)

Summary: This reverts commit d0de55d.

Reviewed By: YosuaMichael

Differential Revision: D40588158

fbshipit-source-id: 877a172d7dd807b0c90255bd14129a90768bcc76

Author: Joao Gomes

.github/workflows/prototype-tests-gpu.yml

@@ -174,10 +174,7 @@ def test_cuda_vs_cpu(self, info, args_kwargs):
         output_cpu = info.kernel(input_cpu, *other_args, **kwargs)
         output_cuda = info.kernel(input_cuda, *other_args, **kwargs)
 
-        try:
-            assert_close(output_cuda, output_cpu, check_device=False, **info.closeness_kwargs)
-        except AssertionError:
-            pytest.xfail("CUDA vs CPU tolerance issue to be fixed")
+        assert_close(output_cuda, output_cpu, check_device=False, **info.closeness_kwargs)
 
     @sample_inputs
     @pytest.mark.parametrize("device", cpu_and_gpu())