8266528: Optimize C2 VerifyIterativeGVN execution time

src/hotspot/share/opto/node.cpp

@@ -332,6 +332,7 @@ Node::Node(uint req)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   assert( req < Compile::current()->max_node_limit() - NodeLimitFudgeFactor, "Input limit exceeded" );
@@ -353,6 +354,7 @@ Node::Node(Node *n0)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -367,6 +369,7 @@ Node::Node(Node *n0, Node *n1)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -383,6 +386,7 @@ Node::Node(Node *n0, Node *n1, Node *n2)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -401,6 +405,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -421,6 +426,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, Node *n4)
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -444,6 +450,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3,
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -469,6 +476,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3,
 #ifdef ASSERT
   , _parse_idx(_idx)
   , _indent(0)
+  , _igvn_verify_epoch(0)
 #endif
 {
   debug_only( verify_construction() );
@@ -2210,61 +2218,122 @@ void Node::verify_edges(Unique_Node_List &visited) {
   }
 }
 
-// Verify all nodes if verify_depth is negative
-void Node::verify(Node* n, int verify_depth) {
-  assert(verify_depth != 0, "depth should not be 0");
+/*
+ * Design target:
+ * 1. Each node is only verified once in PhaseIterGVN::verify_step, optimize
+ *    1.1. Redundant verifications between full and nodes in _verify_window
+ *    1.2. Redundant verifications between nodes in _verify_window
+ * 2. Node's def-use count is only verified once.
+ *
+ * Node none product fields to record IGVN verification status:
+ * 1. _igvn_verify_epoch: if node is visisted in current PhaseIterGVN::verify_step
+ * 2. _igvn_verify_depth_cur: processed depth in current Node::verify
+ * 3. _igvn_verify_depth_prev: processed depth in previous Node::verify
+ *
+ * Status:
+ * 1. _igvn_verify_epoch != verify_epoch: self and recurisive check
+ * 2. _igvn_verify_epoch == verify_epoch:
+ *    2.1. _igvn_verify_depth_prev == 0: self and recurisive check
+ *    2.2. 0 < _igvn_verify_depth_prev < _igvn_verify_depth_cur: recurisive check
+ *
+ * Actions:
+ * Adding node into worklist when:
+ * 1. _igvn_verify_epoch is different with current verify_epoch
+ * 2. Or _igvn_verify_depth_cur is smaller than verify_depth
+ * Note: verify_depth variable decreased in this method.
+ *
+ * When adding node into worklist:
+ * 1. If _igvn_verify_epoch is different with current verify_epoch
+ *    _igvn_verify_depth_prev = 0
+ *    _igvn_verify_depth_cur = (current verify_depth)
+ * 2. Else
+ *    _igvn_verify_depth_prev = _igvn_verify_depth_cur
+ *    _igvn_verify_depth_cur = (current verify_depth)
+ *
+ * As recursive is performed in BFS, bigger verify_depth is processed
+ * before smaller verify_depth, there are no redundannt nodes in worklist.
+ */
+ void Node::verify(Node* n, jint verify_depth, julong verify_epoch) {
+  assert(verify_depth > 0, "depth should not be positive");
+  if (n->_igvn_verify_epoch != verify_epoch) {
+    n->_igvn_verify_epoch = verify_epoch;
+    n->_igvn_verify_depth_prev = 0;
+    n->_igvn_verify_depth_cur = verify_depth;
+  } else if (n->_igvn_verify_depth_cur < verify_depth) {
+    n->_igvn_verify_depth_prev = n->_igvn_verify_depth_cur;
+    n->_igvn_verify_depth_cur = verify_depth;
+  } else {
+    return;
+  }
   ResourceMark rm;
-  VectorSet old_space;
-  VectorSet new_space;
   Node_List worklist;
   worklist.push(n);
   Compile* C = Compile::current();
+  Node* top = C->top();
   uint last_index_on_current_depth = 0;
   verify_depth--; // Visiting the first node on depth 1
-  // Only add nodes to worklist if verify_depth is negative (visit all nodes) or greater than 0
-  bool add_to_worklist = verify_depth != 0;
-
+  bool add_to_worklist = verify_depth > 0; // add nodes to worklist if verify_depth is positive
 
   for (uint list_index = 0; list_index < worklist.size(); list_index++) {
     n = worklist[list_index];
 
-    if (n->is_Con() && n->bottom_type() == Type::TOP) {
+    bool verify_self = (n->_igvn_verify_depth_prev == 0);
+
+    if (verify_self && n->is_Con() && n->bottom_type() == Type::TOP) {
       if (C->cached_top_node() == NULL) {
         C->set_cached_top_node((Node*)n);
       }
       assert(C->cached_top_node() == n, "TOP node must be unique");
     }
 
-    for (uint i = 0; i < n->len(); i++) {
-      Node* x = n->in(i);
-      if (!x || x->is_top()) {
+    uint in_len = n->len();
+    for (uint i = 0; i < in_len; i++) {
+      Node* x = n->_in[i];
+      if (!x || x == top) {
         continue;
       }
 
       // Verify my input has a def-use edge to me
       // Count use-def edges from n to x
-      int cnt = 0;
-      for (uint j = 0; j < n->len(); j++) {
-        if (n->in(j) == x) {
-          cnt++;
+      if (verify_self) {
+        int cnt = 1;
+        for (uint j = 0; j < i; j++) {
+          if (n->_in[j] == x) {
+            cnt++;
+            break;
+          }
+        }
+        // has same input node before i, x already checked
+        if (cnt == 2) {
+          continue;
+        }
+        for (uint j = i + 1; j < in_len; j++) {
+          if (n->_in[j] == x) {
+            cnt++;
+          }
         }
-      }
 
-      // Count def-use edges from x to n
-      uint max = x->_outcnt;
-      for (uint k = 0; k < max; k++) {
-        if (x->_out[k] == n) {
-          cnt--;
+        // Count def-use edges from x to n
+        uint max = x->_outcnt;
+        for (uint k = 0; k < max; k++) {
+          if (x->_out[k] == n) {
+            cnt--;
+          }
         }
+        assert(cnt == 0, "mismatched def-use edge counts");
       }
-      assert(cnt == 0, "mismatched def-use edge counts");
-
-      // Contained in new_space or old_space?
-      VectorSet* v = C->node_arena()->contains(x) ? &new_space : &old_space;
-      // Check for visited in the proper space. Numberings are not unique
-      // across spaces so we need a separate VectorSet for each space.
-      if (add_to_worklist && !v->test_set(x->_idx)) {
-        worklist.push(x);
+
+      if (add_to_worklist) {
+        if (x->_igvn_verify_epoch != verify_epoch) {
+          x->_igvn_verify_epoch = verify_epoch;
+          x->_igvn_verify_depth_prev = 0;
+          x->_igvn_verify_depth_cur = verify_depth;
+          worklist.push(x);
+        } else if (x->_igvn_verify_depth_cur < verify_depth) {
+          x->_igvn_verify_depth_prev = x->_igvn_verify_depth_cur;
+          x->_igvn_verify_depth_cur = verify_depth;
+          worklist.push(x);
+        }
       }
     }
 

src/hotspot/share/opto/node.hpp

@@ -1161,6 +1161,10 @@ class Node {
 #ifndef PRODUCT
  private:
   int _indent;
+  // used in VerifyIterativeGVN, indicate if Node is processed in this iteration.
+  jint _igvn_verify_depth_cur;
+  jint _igvn_verify_depth_prev;
+  julong _igvn_verify_epoch;
 
  public:
   void set_indent(int indent) { _indent = indent; }
@@ -1201,7 +1205,7 @@ class Node {
   void collect_nodes_out_all_ctrl_boundary(GrowableArray<Node*> *ns) const;
 
   void verify_edges(Unique_Node_List &visited); // Verify bi-directional edges
-  static void verify(Node* n, int verify_depth);
+  static void verify(Node* n, jint verify_depth, julong verify_epoch);
 
   // This call defines a class-unique string used to identify class instances
   virtual const char *Name() const;

src/hotspot/share/opto/phaseX.cpp

@@ -967,6 +967,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseIterGVN* igvn) : PhaseGVN(igvn),
                                                  _stack(igvn->_stack ),
                                                  _worklist(igvn->_worklist)
 {
+  NOT_PRODUCT(_verify_epoch = 0;)
   _iterGVN = true;
 }
 
@@ -981,6 +982,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseGVN* gvn) : PhaseGVN(gvn),
                                             _stack(C->comp_arena(), 32),
                                             _worklist(*C->for_igvn())
 {
+  NOT_PRODUCT(_verify_epoch = 0;)
   _iterGVN = true;
   uint max;
 
@@ -1024,11 +1026,13 @@ void PhaseIterGVN::shuffle_worklist() {
 #ifndef PRODUCT
 void PhaseIterGVN::verify_step(Node* n) {
   if (VerifyIterativeGVN) {
+    // Node::verify is only invoked in PhaseIterGVN::verify_step
+    ++_verify_epoch;
     _verify_window[_verify_counter % _verify_window_size] = n;
     ++_verify_counter;
     if (C->unique() < 1000 || 0 == _verify_counter % (C->unique() < 10000 ? 10 : 100)) {
       ++_verify_full_passes;
-      Node::verify(C->root(), -1);
+      Node::verify(C->root(), max_jint, _verify_epoch);
     }
     for (int i = 0; i < _verify_window_size; i++) {
       Node* n = _verify_window[i];
@@ -1041,7 +1045,7 @@ void PhaseIterGVN::verify_step(Node* n) {
         continue;
       }
       // Typical fanout is 1-2, so this call visits about 6 nodes.
-      Node::verify(n, 4);
+      Node::verify(n, 4, _verify_epoch);
     }
   }
 }

src/hotspot/share/opto/phaseX.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -555,6 +555,7 @@ class PhaseIterGVN : public PhaseGVN {
   // Sub-quadratic implementation of VerifyIterativeGVN.
   julong _verify_counter;
   julong _verify_full_passes;
+  julong _verify_epoch;
   enum { _verify_window_size = 30 };
   Node* _verify_window[_verify_window_size];
   void verify_step(Node* n);