8314295: Enhance verification of verifier

Reviewed-by: yan, andrew
Backport-of: 08980a0a60bc48c17eacd57fd2d7065ac2d986a8

Author: martinuy

hotspot/src/share/vm/classfile/verifier.cpp

@@ -2081,11 +2081,12 @@ void ClassVerifier::verify_switch(
           "low must be less than or equal to high in tableswitch");
       return;
     }
-    keys = high - low + 1;
-    if (keys < 0) {
+    int64_t keys64 = ((int64_t)high - low) + 1;
+    if (keys64 > 65535) {  // Max code length
       verify_error(ErrorContext::bad_code(bci), "too many keys in tableswitch");
       return;
     }
+    keys = (int)keys64;
     delta = 1;
   } else {
     keys = (int)Bytes::get_Java_u4(aligned_bcp + jintSize);

hotspot/src/share/vm/interpreter/bytecodes.cpp

@@ -114,12 +114,18 @@ int Bytecodes::special_length_at(Bytecodes::Code code, address bcp, address end)
       if (end != NULL && aligned_bcp + 3*jintSize >= end) {
         return -1; // don't read past end of code buffer
       }
+      // Promote calculation to signed 64 bits to do range checks, used by the verifier.
       jlong lo = (jint)Bytes::get_Java_u4(aligned_bcp + 1*jintSize);
       jlong hi = (jint)Bytes::get_Java_u4(aligned_bcp + 2*jintSize);
       jlong len = (aligned_bcp - bcp) + (3 + hi - lo + 1)*jintSize;
-      // only return len if it can be represented as a positive int;
-      // return -1 otherwise
-      return (len > 0 && len == (int)len) ? len : -1;
+      // Only return len if it can be represented as a positive int and lo <= hi.
+      // The caller checks for bytecode stream overflow.
+      if (lo <= hi && len == (int)len) {
+        assert(len > 0, "must be");
+        return (int)len;
+      } else {
+        return -1;
+      }
     }
 
   case _lookupswitch:      // fall through
@@ -131,9 +137,13 @@ int Bytecodes::special_length_at(Bytecodes::Code code, address bcp, address end)
       }
       jlong npairs = (jint)Bytes::get_Java_u4(aligned_bcp + jintSize);
       jlong len = (aligned_bcp - bcp) + (2 + 2*npairs)*jintSize;
-      // only return len if it can be represented as a positive int;
-      // return -1 otherwise
-      return (len > 0 && len == (int)len) ? len : -1;
+      // Only return len if it can be represented as a positive int and npairs >= 0.
+      if (npairs >= 0 && len == (int)len) {
+        assert(len > 0, "must be");
+        return (int)len;
+      } else {
+        return -1;
+      }
     }
   }
   // Note: Length functions must return <=0 for invalid bytecodes.

jdk/src/share/native/common/check_code.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1994, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -84,6 +84,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #include "jni.h"
 #include "jvm.h"
@@ -1202,7 +1203,7 @@ verify_opcode_operands(context_type *context, unsigned int inumber, int offset)
             }
         }
         if (opcode == JVM_OPC_tableswitch) {
-            keys = _ck_ntohl(lpc[2]) -  _ck_ntohl(lpc[1]) + 1;
+            keys = _ck_ntohl(lpc[2]) - _ck_ntohl(lpc[1]) + 1;
             delta = 1;
         } else {
             keys = _ck_ntohl(lpc[1]); /* number of pairs */
@@ -1682,11 +1683,13 @@ static int instruction_length(unsigned char *iptr, unsigned char *end)
     switch (instruction) {
         case JVM_OPC_tableswitch: {
             int *lpc = (int *)UCALIGN(iptr + 1);
-            int index;
             if (lpc + 2 >= (int *)end) {
                 return -1; /* do not read pass the end */
             }
-            index = _ck_ntohl(lpc[2]) - _ck_ntohl(lpc[1]);
+            int64_t low  = _ck_ntohl(lpc[1]);
+            int64_t high = _ck_ntohl(lpc[2]);
+            int64_t index = high - low;
+            // The value of low must be less than or equal to high - i.e. index >= 0
             if ((index < 0) || (index > 65535)) {
                 return -1;      /* illegal */
             } else {