Skip to content

Commit 3f07e57

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into YARN-7953 
2 parents 0007e2a + 335587d commit 3f07e57

File tree

53 files changed

+2497
-1335
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+2497
-1335
lines changed

‎LICENSE-binary‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -479,8 +479,9 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
479479
com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
480480
com.microsoft.azure:azure-keyvault-core:1.0.0
481481
com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
482-
org.bouncycastle:bcpkix-jdk15on:1.68
483-
org.bouncycastle:bcprov-jdk15on:1.68
482+
org.bouncycastle:bcpkix-jdk15on:1.70
483+
org.bouncycastle:bcprov-jdk15on:1.70
484+
org.bouncycastle:bcutil-jdk15on:1.70
484485
org.checkerframework:checker-qual:2.5.2
485486
org.codehaus.mojo:animal-sniffer-annotations:1.21
486487
org.jruby.jcodings:jcodings:1.0.13

‎dev-support/jenkinsfile-windows-10‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
def getGithubCreds() {
19+
return [usernamePassword(credentialsId: 'apache-hadoop-at-github.202132.xyz',
20+
passwordVariable: 'GITHUB_TOKEN',
21+
usernameVariable: 'GITHUB_USER')]
22+
}
23+
24+
// Publish JUnit results only if there are XML files under surefire-reports
25+
def publishJUnitResults() {
26+
def findCmdExitCode = sh script: "find ${SOURCEDIR} -wholename */target/surefire-reports/*.xml | egrep .", returnStatus: true
27+
boolean surefireReportsExist = findCmdExitCode == 0
28+
if (surefireReportsExist) {
29+
echo "XML files found under surefire-reports, running junit"
30+
// The path should be relative to WORKSPACE for the junit.
31+
SRC = "${SOURCEDIR}/**/target/surefire-reports/*.xml".replace("$WORKSPACE/","")
32+
try {
33+
junit "${SRC}"
34+
} catch(e) {
35+
echo 'junit processing: ' + e.toString()
36+
}
37+
} else {
38+
echo "No XML files found under surefire-reports, skipping junit"
39+
}
40+
}
41+
42+
pipeline {
43+
agent {
44+
label 'Windows'
45+
}
46+
47+
options {
48+
buildDiscarder(logRotator(numToKeepStr: '15'))
49+
timeout (time: 72, unit: 'HOURS')
50+
timestamps()
51+
checkoutToSubdirectory('src')
52+
}
53+
54+
environment {
55+
YETUS='yetus'
56+
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
57+
YETUS_VERSION='rel/0.14.0'
58+
}
59+
60+
parameters {
61+
string(name: 'JIRA_ISSUE_KEY',
62+
defaultValue: '',
63+
description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
64+
}
65+
66+
stages {
67+
stage ('Install Yetus') {
68+
steps {
69+
dir("${WORKSPACE}/${YETUS}") {
70+
checkout([
71+
$class: 'GitSCM',
72+
branches: [[name: "${env.YETUS_VERSION}"]],
73+
userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
74+
)
75+
}
76+
}
77+
}
78+
79+
stage ('Windows 10') {
80+
environment {
81+
SOURCEDIR = "${WORKSPACE}/src"
82+
PATCHDIR = "${WORKSPACE}/out"
83+
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_windows_10"
84+
DOCKER_BUILDKIT = 0
85+
IS_OPTIONAL = 0
86+
IS_NIGHTLY_BUILD = 0
87+
IS_WINDOWS = 1
88+
}
89+
90+
steps {
91+
withCredentials(getGithubCreds()) {
92+
bat 'if not exist %WORKSPACE%\\out mkdir %WORKSPACE%\\out'
93+
bat 'if not exist F:\\maven\\repository mkdir F:\\maven\\repository'
94+
bat '''docker build --label org.apache.yetus=""^
95+
--label org.apache.yetus.testpatch.project=hadoop^
96+
--tag hadoop-windows-10-builder^
97+
-f %DOCKERFILE% %SOURCEDIR%\\dev-support\\docker'''
98+
bat '''docker run --rm -v %WORKSPACE%\\out:C:\\out^
99+
-v %WORKSPACE%\\src:C:\\src^
100+
-v %WORKSPACE%\\yetus:C:\\yetus^
101+
-v F:\\maven\\repository:C:\\Users\\%BUILD_USER%\\.m2^
102+
-e BUILD_URL=%BUILD_URL% -e BRANCH_NAME=%BRANCH_NAME%^
103+
-e JOB_NAME=%JOB_NAME% -e JENKINS_URL=%JENKINS_URL%^
104+
-e CHANGE_URL=%CHANGE_URL% -e WORKSPACE=/c -e YETUS=yetus^
105+
-e BUILD_NUMBER=%BUILD_NUMBER% -e EXECUTOR_NUMBER=%EXECUTOR_NUMBER%^
106+
-e BUILD_ID=%BUILD_ID% -e GIT_COMMIT=%GIT_COMMIT%^
107+
-e NODE_NAME=%NODE_NAME% -e GIT_URL=%GIT_URL%^
108+
-e BUILD_TAG=%BUILD_TAG% -e GIT_BRANCH=%GIT_BRANCH%^
109+
-e PATCH_OR_ISSUE=%CHANGE_URL% -e JIRA_ISSUE_KEY=%JIRA_ISSUE_KEY%^
110+
-e IS_OPTIONAL=0 -e SOURCEDIR=/c/hadoop -e PATCHDIR=/c/out^
111+
-e GITHUB_TOKEN=%GITHUB_TOKEN% -e GITHUB_USER=%GITHUB_USER%^
112+
-e IS_NIGHTLY_BUILD=%IS_NIGHTLY_BUILD% -e IS_WINDOWS=%IS_WINDOWS%^
113+
-e BASH_EXECUTABLE=/c/Git/bin/bash.exe^
114+
-e VCPKG_INSTALLED_PACKAGES=/c/vcpkg/installed/x64-windows^
115+
-e CMAKE_TOOLCHAIN_FILE=/c/vcpkg/scripts/buildsystems/vcpkg.cmake^
116+
hadoop-windows-10-builder "/c" "xcopy" "/s" "/e" "/h" "/y" "/i" "/q" "C:\\src" "C:\\hadoop"^
117+
"&&" "C:\\Git\\bin\\bash.exe" "-c" \'"/c/src/dev-support/jenkins.sh" "run_ci"\'
118+
'''
119+
}
120+
}
121+
122+
post {
123+
always {
124+
script {
125+
bat '''docker run --rm --user ContainerAdministrator^
126+
-v %WORKSPACE%\\out:C:\\out^
127+
hadoop-windows-10-builder "/c" "icacls" "C:\\out" "/c" "/t" "/q" "/grant" "Users:F"
128+
'''
129+
bat '''docker run --rm --user ContainerAdministrator^
130+
-v F:\\maven\\repository:C:\\Users\\%BUILD_USER%\\.m2^
131+
hadoop-windows-10-builder "/c" "icacls" "C:\\Users\\%BUILD_USER%\\.m2" "/c" "/t" "/q" "/grant" "Users:F"
132+
'''
133+
134+
// Publish status if it was missed (YETUS-1059)
135+
withCredentials(
136+
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
137+
passwordVariable: 'GITHUB_TOKEN',
138+
usernameVariable: 'GITHUB_USER')]) {
139+
bat '''docker run --rm -v %WORKSPACE%\\out:C:\\out^
140+
-v %WORKSPACE%\\src:C:\\src^
141+
-v %WORKSPACE%\\yetus:C:\\yetus^
142+
-v F:\\maven\\repository:C:\\Users\\%BUILD_USER%\\.m2^
143+
-e BUILD_URL=%BUILD_URL% -e BRANCH_NAME=%BRANCH_NAME%^
144+
-e JOB_NAME=%JOB_NAME% -e JENKINS_URL=%JENKINS_URL%^
145+
-e CHANGE_URL=%CHANGE_URL% -e WORKSPACE=/c -e YETUS=yetus^
146+
-e BUILD_NUMBER=%BUILD_NUMBER% -e EXECUTOR_NUMBER=%EXECUTOR_NUMBER%^
147+
-e BUILD_ID=%BUILD_ID% -e GIT_COMMIT=%GIT_COMMIT%^
148+
-e NODE_NAME=%NODE_NAME% -e GIT_URL=%GIT_URL%^
149+
-e BUILD_TAG=%BUILD_TAG% -e GIT_BRANCH=%GIT_BRANCH%^
150+
-e PATCH_OR_ISSUE=%CHANGE_URL% -e JIRA_ISSUE_KEY=%JIRA_ISSUE_KEY%^
151+
-e IS_OPTIONAL=0 -e SOURCEDIR=/c/hadoop -e PATCHDIR=/c/out^
152+
-e GITHUB_TOKEN=%GITHUB_TOKEN% -e GITHUB_USER=%GITHUB_USER%^
153+
-e IS_NIGHTLY_BUILD=%IS_NIGHTLY_BUILD% -e IS_WINDOWS=%IS_WINDOWS%^
154+
-e BASH_EXECUTABLE=/c/Git/bin/bash.exe^
155+
-e VCPKG_INSTALLED_PACKAGES=/c/vcpkg/installed/x64-windows^
156+
-e CMAKE_TOOLCHAIN_FILE=/c/vcpkg/scripts/buildsystems/vcpkg.cmake^
157+
hadoop-windows-10-builder "/c"^
158+
"C:\\Git\\bin\\bash.exe" "-c" \'"/c/src/dev-support/jenkins.sh" "github_status_recovery"\'
159+
'''
160+
}
161+
162+
archiveArtifacts "out/**"
163+
164+
publishJUnitResults()
165+
166+
// Publish the HTML report so that it can be looked at
167+
// Has to be relative to WORKSPACE.
168+
publishHTML (target: [
169+
allowMissing: true,
170+
keepAll: true,
171+
alwaysLinkToLastBuild: true,
172+
// Has to be relative to WORKSPACE
173+
reportDir: "out",
174+
reportFiles: 'report.html',
175+
reportName: 'Yetus Report'
176+
])
177+
}
178+
}
179+
180+
cleanup() {
181+
script {
182+
bat '''
183+
C:\\PROGRA~1\\Git\\bin\\bash.exe -c "chmod u+x ${SOURCEDIR}/dev-support/jenkins.sh"
184+
C:\\PROGRA~1\\Git\\bin\\bash.exe -c "${SOURCEDIR}/dev-support/jenkins.sh cleanup_ci_proc"
185+
'''
186+
}
187+
}
188+
}
189+
}
190+
}
191+
192+
post {
193+
// Jenkins pipeline jobs fill slaves on PRs without this :(
194+
cleanup() {
195+
script {
196+
deleteDir()
197+
}
198+
}
199+
}
200+
}

‎hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
425425
| `RamDiskBlocksLazyPersistWindows`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of latency between memory write and disk persist in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
426426
| `FsyncCount` | Total number of fsync |
427427
| `VolumeFailures` | Total number of volume failures occurred |
428+
| `DatanodeNetworkErrors` | Count of network errors on the datanode |
429+
| `DataNodeActiveXceiversCount` | Count of active dataNode xceivers |
430+
| `DataNodeReadActiveXceiversCount` | Count of read active dataNode xceivers |
431+
| `DataNodeWriteActiveXceiversCount` | Count of write active dataNode xceivers |
432+
| `DataNodePacketResponderCount` | Count of active DataNode packetResponder |
433+
| `DataNodeBlockRecoveryWorkerCount` | Count of active DataNode block recovery worker |
428434
| `ReadBlockOpNumOps` | Total number of read operations |
429435
| `ReadBlockOpAvgTime` | Average time of read operations in milliseconds |
430436
| `WriteBlockOpNumOps` | Total number of write operations |

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1864,11 +1864,11 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
18641864
maxECReplicatedTransfers = maxTransfers;
18651865
}
18661866
int numReplicationTasks = (int) Math.ceil(
1867-
(double) (replicationBlocks * maxTransfers) / totalBlocks);
1867+
(double) replicationBlocks * maxTransfers / totalBlocks);
18681868
int numEcReplicatedTasks = (int) Math.ceil(
1869-
(double) (ecBlocksToBeReplicated * maxECReplicatedTransfers) / totalBlocks);
1869+
(double) ecBlocksToBeReplicated * maxECReplicatedTransfers / totalBlocks);
18701870
int numECReconstructedTasks = (int) Math.ceil(
1871-
(double) (ecBlocksToBeErasureCoded * maxTransfers) / totalBlocks);
1871+
(double) ecBlocksToBeErasureCoded * maxTransfers / totalBlocks);
18721872
LOG.debug("Pending replication tasks: {} ec to be replicated tasks: {} " +
18731873
"ec reconstruction tasks: {}.",
18741874
numReplicationTasks, numEcReplicatedTasks, numECReconstructedTasks);

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2639,6 +2639,8 @@ public void shutdown() {
26392639
}
26402640
if (metrics != null) {
26412641
metrics.setDataNodeActiveXceiversCount(0);
2642+
metrics.setDataNodeReadActiveXceiversCount(0);
2643+
metrics.setDataNodeWriteActiveXceiversCount(0);
26422644
metrics.setDataNodePacketResponderCount(0);
26432645
metrics.setDataNodeBlockRecoveryWorkerCount(0);
26442646
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ private synchronized void putThreadName(String thread) {
256256
threadCountMap.putIfAbsent(thread, new TrackLog(thread));
257257
}
258258

259-
public void lockLeakCheck() {
259+
public synchronized void lockLeakCheck() {
260260
if (!openLockTrace) {
261261
LOG.warn("not open lock leak check func");
262262
return;

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ private synchronized BlockReceiver getCurrentBlockReceiver() {
223223
public void run() {
224224
int opsProcessed = 0;
225225
Op op = null;
226+
Op firstOp = null;
226227

227228
try {
228229
synchronized(this) {
@@ -290,6 +291,11 @@ public void run() {
290291
}
291292

292293
opStartTime = monotonicNow();
294+
// compatible with loop retry requests
295+
if (firstOp == null) {
296+
firstOp = op;
297+
incrReadWriteOpMetrics(op);
298+
}
293299
processOp(op);
294300
++opsProcessed;
295301
} while ((peer != null) &&
@@ -330,6 +336,9 @@ public void run() {
330336
datanode.getDisplayName(), datanode.getXceiverCount());
331337
updateCurrentThreadName("Cleaning up");
332338
if (peer != null) {
339+
if (firstOp != null) {
340+
decrReadWriteOpMetrics(op);
341+
}
333342
dataXceiverServer.closePeer(peer);
334343
IOUtils.closeStream(in);
335344
}
@@ -1466,4 +1475,20 @@ private void checkAccess(OutputStream out, final boolean reply,
14661475
}
14671476
}
14681477
}
1478+
1479+
private void incrReadWriteOpMetrics(Op op) {
1480+
if (Op.READ_BLOCK.equals(op)) {
1481+
datanode.getMetrics().incrDataNodeReadActiveXceiversCount();
1482+
} else if (Op.WRITE_BLOCK.equals(op)) {
1483+
datanode.getMetrics().incrDataNodeWriteActiveXceiversCount();
1484+
}
1485+
}
1486+
1487+
private void decrReadWriteOpMetrics(Op op) {
1488+
if (Op.READ_BLOCK.equals(op)) {
1489+
datanode.getMetrics().decrDataNodeReadActiveXceiversCount();
1490+
} else if (Op.WRITE_BLOCK.equals(op)) {
1491+
datanode.getMetrics().decrDataNodeWriteActiveXceiversCount();
1492+
}
1493+
}
14691494
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,8 @@ void closeAllPeers() {
413413
peers.clear();
414414
peersXceiver.clear();
415415
datanode.metrics.setDataNodeActiveXceiversCount(0);
416+
datanode.metrics.setDataNodeReadActiveXceiversCount(0);
417+
datanode.metrics.setDataNodeWriteActiveXceiversCount(0);
416418
this.noPeers.signalAll();
417419
} finally {
418420
lock.unlock();

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ public class DataNodeMetrics {
111111
@Metric("Count of active dataNode xceivers")
112112
private MutableGaugeInt dataNodeActiveXceiversCount;
113113

114+
@Metric("Count of read active dataNode xceivers")
115+
private MutableGaugeInt dataNodeReadActiveXceiversCount;
116+
117+
@Metric("Count of write active dataNode xceivers")
118+
private MutableGaugeInt dataNodeWriteActiveXceiversCount;
119+
114120
@Metric("Count of active DataNode packetResponder")
115121
private MutableGaugeInt dataNodePacketResponderCount;
116122

@@ -599,6 +605,30 @@ public int getDataNodeActiveXceiverCount() {
599605
return dataNodeActiveXceiversCount.value();
600606
}
601607

608+
public void incrDataNodeReadActiveXceiversCount(){
609+
dataNodeReadActiveXceiversCount.incr();
610+
}
611+
612+
public void decrDataNodeReadActiveXceiversCount(){
613+
dataNodeReadActiveXceiversCount.decr();
614+
}
615+
616+
public void setDataNodeReadActiveXceiversCount(int value){
617+
dataNodeReadActiveXceiversCount.set(value);
618+
}
619+
620+
public void incrDataNodeWriteActiveXceiversCount(){
621+
dataNodeWriteActiveXceiversCount.incr();
622+
}
623+
624+
public void decrDataNodeWriteActiveXceiversCount(){
625+
dataNodeWriteActiveXceiversCount.decr();
626+
}
627+
628+
public void setDataNodeWriteActiveXceiversCount(int value){
629+
dataNodeWriteActiveXceiversCount.set(value);
630+
}
631+
602632
public void incrDataNodePacketResponderCount() {
603633
dataNodePacketResponderCount.incr();
604634
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3815,7 +3815,10 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip,
38153815
// may be client have crashed before writing data to pipeline.
38163816
// This blocks doesn't need any recovery.
38173817
// We can remove this block and close the file.
3818-
pendingFile.removeLastBlock(lastBlock);
3818+
BlockInfo lastBlockInfo = pendingFile.removeLastBlock(lastBlock);
3819+
if (lastBlockInfo != null) {
3820+
blockManager.removeBlock(lastBlockInfo);
3821+
}
38193822
finalizeINodeFileUnderConstruction(src, pendingFile,
38203823
iip.getLatestSnapshotId(), false);
38213824
if (uc.getNumExpectedLocations() == 0) {

0 commit comments

Comments
 (0)