From 11842903bc934375a637e65f4c1e1c8946df9075 Mon Sep 17 00:00:00 2001 From: kalinshi Date: Wed, 5 May 2021 10:03:21 +0800 Subject: [PATCH] 8266528: Optimize C2 VerifyIterativeGVN execution time --- src/hotspot/share/opto/node.cpp | 129 +++++++++++++++++++++++------- src/hotspot/share/opto/node.hpp | 6 +- src/hotspot/share/opto/phaseX.cpp | 8 +- src/hotspot/share/opto/phaseX.hpp | 3 +- 4 files changed, 112 insertions(+), 34 deletions(-) diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index b671c4479b127..86244b2680119 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -332,6 +332,7 @@ Node::Node(uint req) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { assert( req < Compile::current()->max_node_limit() - NodeLimitFudgeFactor, "Input limit exceeded" ); @@ -353,6 +354,7 @@ Node::Node(Node *n0) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -367,6 +369,7 @@ Node::Node(Node *n0, Node *n1) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -383,6 +386,7 @@ Node::Node(Node *n0, Node *n1, Node *n2) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -401,6 +405,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -421,6 +426,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, Node *n4) #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -444,6 +450,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -469,6 +476,7 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, #ifdef ASSERT , _parse_idx(_idx) , _indent(0) + , _igvn_verify_epoch(0) #endif { debug_only( verify_construction() ); @@ -2210,61 +2218,122 @@ void Node::verify_edges(Unique_Node_List &visited) { } } -// Verify all nodes if verify_depth is negative -void Node::verify(Node* n, int verify_depth) { - assert(verify_depth != 0, "depth should not be 0"); +/* + * Design target: + * 1. Each node is only verified once in PhaseIterGVN::verify_step, optimize + * 1.1. Redundant verifications between full and nodes in _verify_window + * 1.2. Redundant verifications between nodes in _verify_window + * 2. Node's def-use count is only verified once. + * + * Node none product fields to record IGVN verification status: + * 1. _igvn_verify_epoch: if node is visisted in current PhaseIterGVN::verify_step + * 2. _igvn_verify_depth_cur: processed depth in current Node::verify + * 3. _igvn_verify_depth_prev: processed depth in previous Node::verify + * + * Status: + * 1. _igvn_verify_epoch != verify_epoch: self and recurisive check + * 2. _igvn_verify_epoch == verify_epoch: + * 2.1. _igvn_verify_depth_prev == 0: self and recurisive check + * 2.2. 0 < _igvn_verify_depth_prev < _igvn_verify_depth_cur: recurisive check + * + * Actions: + * Adding node into worklist when: + * 1. _igvn_verify_epoch is different with current verify_epoch + * 2. Or _igvn_verify_depth_cur is smaller than verify_depth + * Note: verify_depth variable decreased in this method. + * + * When adding node into worklist: + * 1. If _igvn_verify_epoch is different with current verify_epoch + * _igvn_verify_depth_prev = 0 + * _igvn_verify_depth_cur = (current verify_depth) + * 2. Else + * _igvn_verify_depth_prev = _igvn_verify_depth_cur + * _igvn_verify_depth_cur = (current verify_depth) + * + * As recursive is performed in BFS, bigger verify_depth is processed + * before smaller verify_depth, there are no redundannt nodes in worklist. + */ + void Node::verify(Node* n, jint verify_depth, julong verify_epoch) { + assert(verify_depth > 0, "depth should not be positive"); + if (n->_igvn_verify_epoch != verify_epoch) { + n->_igvn_verify_epoch = verify_epoch; + n->_igvn_verify_depth_prev = 0; + n->_igvn_verify_depth_cur = verify_depth; + } else if (n->_igvn_verify_depth_cur < verify_depth) { + n->_igvn_verify_depth_prev = n->_igvn_verify_depth_cur; + n->_igvn_verify_depth_cur = verify_depth; + } else { + return; + } ResourceMark rm; - VectorSet old_space; - VectorSet new_space; Node_List worklist; worklist.push(n); Compile* C = Compile::current(); + Node* top = C->top(); uint last_index_on_current_depth = 0; verify_depth--; // Visiting the first node on depth 1 - // Only add nodes to worklist if verify_depth is negative (visit all nodes) or greater than 0 - bool add_to_worklist = verify_depth != 0; - + bool add_to_worklist = verify_depth > 0; // add nodes to worklist if verify_depth is positive for (uint list_index = 0; list_index < worklist.size(); list_index++) { n = worklist[list_index]; - if (n->is_Con() && n->bottom_type() == Type::TOP) { + bool verify_self = (n->_igvn_verify_depth_prev == 0); + + if (verify_self && n->is_Con() && n->bottom_type() == Type::TOP) { if (C->cached_top_node() == NULL) { C->set_cached_top_node((Node*)n); } assert(C->cached_top_node() == n, "TOP node must be unique"); } - for (uint i = 0; i < n->len(); i++) { - Node* x = n->in(i); - if (!x || x->is_top()) { + uint in_len = n->len(); + for (uint i = 0; i < in_len; i++) { + Node* x = n->_in[i]; + if (!x || x == top) { continue; } // Verify my input has a def-use edge to me // Count use-def edges from n to x - int cnt = 0; - for (uint j = 0; j < n->len(); j++) { - if (n->in(j) == x) { - cnt++; + if (verify_self) { + int cnt = 1; + for (uint j = 0; j < i; j++) { + if (n->_in[j] == x) { + cnt++; + break; + } + } + // has same input node before i, x already checked + if (cnt == 2) { + continue; + } + for (uint j = i + 1; j < in_len; j++) { + if (n->_in[j] == x) { + cnt++; + } } - } - // Count def-use edges from x to n - uint max = x->_outcnt; - for (uint k = 0; k < max; k++) { - if (x->_out[k] == n) { - cnt--; + // Count def-use edges from x to n + uint max = x->_outcnt; + for (uint k = 0; k < max; k++) { + if (x->_out[k] == n) { + cnt--; + } } + assert(cnt == 0, "mismatched def-use edge counts"); } - assert(cnt == 0, "mismatched def-use edge counts"); - - // Contained in new_space or old_space? - VectorSet* v = C->node_arena()->contains(x) ? &new_space : &old_space; - // Check for visited in the proper space. Numberings are not unique - // across spaces so we need a separate VectorSet for each space. - if (add_to_worklist && !v->test_set(x->_idx)) { - worklist.push(x); + + if (add_to_worklist) { + if (x->_igvn_verify_epoch != verify_epoch) { + x->_igvn_verify_epoch = verify_epoch; + x->_igvn_verify_depth_prev = 0; + x->_igvn_verify_depth_cur = verify_depth; + worklist.push(x); + } else if (x->_igvn_verify_depth_cur < verify_depth) { + x->_igvn_verify_depth_prev = x->_igvn_verify_depth_cur; + x->_igvn_verify_depth_cur = verify_depth; + worklist.push(x); + } } } diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 3fd0cd7444de7..e6e583f8df9c4 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -1161,6 +1161,10 @@ class Node { #ifndef PRODUCT private: int _indent; + // used in VerifyIterativeGVN, indicate if Node is processed in this iteration. + jint _igvn_verify_depth_cur; + jint _igvn_verify_depth_prev; + julong _igvn_verify_epoch; public: void set_indent(int indent) { _indent = indent; } @@ -1201,7 +1205,7 @@ class Node { void collect_nodes_out_all_ctrl_boundary(GrowableArray *ns) const; void verify_edges(Unique_Node_List &visited); // Verify bi-directional edges - static void verify(Node* n, int verify_depth); + static void verify(Node* n, jint verify_depth, julong verify_epoch); // This call defines a class-unique string used to identify class instances virtual const char *Name() const; diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 8c0a7de2d66e5..f4fef993d1525 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -967,6 +967,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseIterGVN* igvn) : PhaseGVN(igvn), _stack(igvn->_stack ), _worklist(igvn->_worklist) { + NOT_PRODUCT(_verify_epoch = 0;) _iterGVN = true; } @@ -981,6 +982,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseGVN* gvn) : PhaseGVN(gvn), _stack(C->comp_arena(), 32), _worklist(*C->for_igvn()) { + NOT_PRODUCT(_verify_epoch = 0;) _iterGVN = true; uint max; @@ -1024,11 +1026,13 @@ void PhaseIterGVN::shuffle_worklist() { #ifndef PRODUCT void PhaseIterGVN::verify_step(Node* n) { if (VerifyIterativeGVN) { + // Node::verify is only invoked in PhaseIterGVN::verify_step + ++_verify_epoch; _verify_window[_verify_counter % _verify_window_size] = n; ++_verify_counter; if (C->unique() < 1000 || 0 == _verify_counter % (C->unique() < 10000 ? 10 : 100)) { ++_verify_full_passes; - Node::verify(C->root(), -1); + Node::verify(C->root(), max_jint, _verify_epoch); } for (int i = 0; i < _verify_window_size; i++) { Node* n = _verify_window[i]; @@ -1041,7 +1045,7 @@ void PhaseIterGVN::verify_step(Node* n) { continue; } // Typical fanout is 1-2, so this call visits about 6 nodes. - Node::verify(n, 4); + Node::verify(n, 4, _verify_epoch); } } } diff --git a/src/hotspot/share/opto/phaseX.hpp b/src/hotspot/share/opto/phaseX.hpp index fe4b199bb7bb6..bea8cf164b256 100644 --- a/src/hotspot/share/opto/phaseX.hpp +++ b/src/hotspot/share/opto/phaseX.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -555,6 +555,7 @@ class PhaseIterGVN : public PhaseGVN { // Sub-quadratic implementation of VerifyIterativeGVN. julong _verify_counter; julong _verify_full_passes; + julong _verify_epoch; enum { _verify_window_size = 30 }; Node* _verify_window[_verify_window_size]; void verify_step(Node* n);