Skip to content

Commit 6f93e93

Browse files
Apache9Apache9
committed
HBASE-28679 Upgrade yetus to a newer version (#6012)
Signed-off-by: Nick Dimiduk <[email protected]> (cherry picked from commit 17ce7c3)
1 parent 9942a7d commit 6f93e93

File tree

5 files changed

+38
-33
lines changed

5 files changed

+38
-33
lines changed

‎dev-support/Jenkinsfile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ pipeline {
3131
disableConcurrentBuilds()
3232
}
3333
environment {
34-
YETUS_RELEASE = '0.12.0'
34+
YETUS_RELEASE = '0.15.0'
3535
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
3636
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
3737
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -42,12 +42,12 @@ pipeline {
4242
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
4343
PERSONALITY_FILE = 'tools/personality.sh'
4444
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
45-
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
46-
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
45+
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
46+
BLANKS_IGNORE_LIST = '.*/generated/.*'
4747
// output from surefire; sadly the archive function in yetus only works on file names.
4848
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
4949
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
50-
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
50+
TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
5151
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
5252
// TODO does hadoopcheck need to be jdk specific?
5353
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'

‎dev-support/Jenkinsfile_GitHub‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@ pipeline {
3636
YETUS_REL = 'yetus'
3737
DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
3838
YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
39-
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
40-
YETUS_VERSION = 'rel/0.12.0'
39+
YETUS_VERSION = '0.15.0'
4140
GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
4241
JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
42+
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
43+
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
44+
BLANKS_IGNORE_LIST = '.*/generated/.*'
4345
// output from surefire; sadly the archive function in yetus only works on file names.
4446
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
4547
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
46-
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
48+
TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
4749
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
4850
// set build parallel
4951
BUILD_THREAD = 4
@@ -101,11 +103,11 @@ pipeline {
101103
checkout scm
102104
}
103105
dir("${YETUSDIR}") {
104-
checkout([
105-
$class : 'GitSCM',
106-
branches : [[name: "${YETUS_VERSION}"]],
107-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
108-
)
106+
sh'''#!/usr/bin/env bash
107+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
108+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
109+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
110+
'''
109111
}
110112
dir("${WORKDIR}") {
111113
withCredentials([
@@ -228,11 +230,11 @@ pipeline {
228230
checkout scm
229231
}
230232
dir("${YETUSDIR}") {
231-
checkout([
232-
$class : 'GitSCM',
233-
branches : [[name: "${YETUS_VERSION}"]],
234-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
235-
)
233+
sh'''#!/usr/bin/env bash
234+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
235+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
236+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
237+
'''
236238
}
237239
dir("${WORKDIR}") {
238240
withCredentials([
@@ -365,11 +367,11 @@ pipeline {
365367
checkout scm
366368
}
367369
dir("${YETUSDIR}") {
368-
checkout([
369-
$class : 'GitSCM',
370-
branches : [[name: "${YETUS_VERSION}"]],
371-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
372-
)
370+
sh'''#!/usr/bin/env bash
371+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
372+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
373+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
374+
'''
373375
}
374376
dir("${WORKDIR}") {
375377
withCredentials([

‎dev-support/docker/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
5353
ruby=1:3.0* \
5454
ruby-dev=1:3.0* \
5555
shellcheck='0.8.0-*' \
56+
libxml2-dev='2.9.13+dfsg-*' \
57+
libxml2-utils='2.9.13+dfsg-*' \
5658
&& \
5759
apt-get clean && \
5860
rm -rf /var/lib/apt/lists/* \

‎dev-support/hbase_nightly_yetus.sh‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ declare -i missing_env=0
2020
# Validate params
2121
for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
2222
"OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
23-
"WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
23+
"BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
2424
"USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
2525
if [ -z "${!required_env}" ]; then
2626
echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
5959
YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
6060
YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
6161
YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
62-
YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
63-
YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
62+
YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
63+
YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
6464
YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
6565
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
6666
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
106106
fi
107107

108108
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
109-
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
110109
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
111110
else
112-
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}")
113-
TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
111+
TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
114112
fi
115113
echo "Launching yetus with command line:"
116114
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

‎dev-support/jenkins_precommit_github_yetus.sh‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ declare -a required_envs=(
4141
"SOURCEDIR"
4242
"TESTS_FILTER"
4343
"YETUSDIR"
44+
"AUTHOR_IGNORE_LIST"
45+
"BLANKS_IGNORE_LIST"
4446
)
4547
# Validate params
4648
for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
5759
fi
5860

5961
# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
60-
TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
62+
TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
6163

6264
# this must be clean for every run
6365
rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
8789
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
8890
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
8991
# enable writing back to Github
90-
YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
91-
YETUS_ARGS+=("--github-user=${GITHUB_USER}")
92+
YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
93+
YETUS_ARGS+=("--github-write-comment")
9294
# auto-kill any surefire stragglers during unit test runs
9395
YETUS_ARGS+=("--reapermode=kill")
9496
# set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
109111
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
110112
YETUS_ARGS+=("--mvn-custom-repos")
111113
YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
112-
YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
113-
YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
114+
YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
115+
YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
116+
YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
114117
YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
115118
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
116119
YETUS_ARGS+=("--quick-hadoopcheck")

0 commit comments

Comments
 (0)