Hdfs ec

BUILDING.txt

@@ -4,7 +4,7 @@ Build instructions for Hadoop
 Requirements:
 
 * Unix System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
@@ -204,15 +204,19 @@ Building on Windows
 Requirements:
 
 * Windows System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
 * CMake 2.6 or newer
 * Windows SDK or Visual Studio 2010 Professional
-* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
 * zlib headers (if building native code bindings for zlib)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
+  tools must be present on your PATH.
+
+Unix command-line tools are also included with the Windows Git package which
+can be downloaded from http://git-scm.com/download/win.
 
 If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
 Do not use Visual Studio Express.  It does not support compiling for 64-bit,
@@ -221,6 +225,8 @@ download here:
 
 http://www.microsoft.com/en-us/download/details.aspx?id=8279
 
+Cygwin is neither required nor supported.
+
 ----------------------------------------------------------------------------------
 Building:
 

dev-support/determine-flaky-tests-hadoop.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Given a jenkins test job, this script examines all runs of the job done
+# within specified period of time (number of days prior to the execution
+# time of this script), and reports all failed tests.
+#
+# The output of this script includes a section for each run that has failed
+# tests, with each failed test name listed.
+#
+# More importantly, at the end, it outputs a summary section to list all failed
+# tests within all examined runs, and indicate how many runs a same test
+# failed, and sorted all failed tests by how many runs each test failed.
+#
+# This way, when we see failed tests in PreCommit build, we can quickly tell
+# whether a failed test is a new failure, or it failed before and how often it
+# failed, so to have idea whether it may just be a flaky test.
+#
+# Of course, to be 100% sure about the reason of a test failure, closer look
+# at the failed test for the specific run is necessary.
+#
+import sys
+import platform
+sysversion = sys.hexversion
+onward30 = False
+if sysversion < 0x020600F0:
+  sys.exit("Minimum supported python version is 2.6, the current version is " +
+      "Python" + platform.python_version())
+
+if sysversion == 0x030000F0:
+  sys.exit("There is a known bug with Python" + platform.python_version() +
+      ", please try a different version");
+
+if sysversion < 0x03000000:
+  import urllib2
+else:
+  onward30 = True
+  import urllib.request
+
+import datetime
+import json as simplejson
+import logging
+from optparse import OptionParser
+import time
+
+# Configuration
+DEFAULT_JENKINS_URL = "https://builds.apache.org"
+DEFAULT_JOB_NAME = "Hadoop-Common-trunk"
+DEFAULT_NUM_PREVIOUS_DAYS = 14
+
+SECONDS_PER_DAY = 86400
+
+# total number of runs to examine
+numRunsToExamine = 0
+
+""" Parse arguments """
+def parse_args():
+  parser = OptionParser()
+  parser.add_option("-J", "--jenkins-url", type="string",
+                    dest="jenkins_url", help="Jenkins URL",
+                    default=DEFAULT_JENKINS_URL)
+  parser.add_option("-j", "--job-name", type="string",
+                    dest="job_name", help="Job name to look at",
+                    default=DEFAULT_JOB_NAME)
+  parser.add_option("-n", "--num-days", type="int",
+                    dest="num_prev_days", help="Number of days to examine",
+                    default=DEFAULT_NUM_PREVIOUS_DAYS)
+
+  (options, args) = parser.parse_args()
+  if args:
+    parser.error("unexpected arguments: " + repr(args))
+  return options
+
+""" Load data from specified url """
+def load_url_data(url):
+  if onward30:
+    ourl = urllib.request.urlopen(url)
+    codec = ourl.info().get_param('charset')
+    content = ourl.read().decode(codec)
+    data = simplejson.loads(content, strict=False)
+  else:
+    ourl = urllib2.urlopen(url)
+    data = simplejson.load(ourl, strict=False)
+  return data
+
+""" List all builds of the target project. """
+def list_builds(jenkins_url, job_name):
+  url = "%(jenkins)s/job/%(job_name)s/api/json?tree=builds[url,result,timestamp]" % dict(
+      jenkins=jenkins_url,
+      job_name=job_name)
+
+  try:
+    data = load_url_data(url)
+
+  except:
+    logging.error("Could not fetch: %s" % url)
+    raise
+  return data['builds']
+
+""" Find the names of any tests which failed in the given build output URL. """
+def find_failing_tests(testReportApiJson, jobConsoleOutput):
+  ret = set()
+  try:
+    data = load_url_data(testReportApiJson)
+
+  except:
+    logging.error("    Could not open testReport, check " +
+        jobConsoleOutput + " for why it was reported failed")
+    return ret
+
+  for suite in data['suites']:
+    for cs in suite['cases']:
+      status = cs['status']
+      errDetails = cs['errorDetails']
+      if (status == 'REGRESSION' or status == 'FAILED' or (errDetails is not None)):
+        ret.add(cs['className'] + "." + cs['name'])
+
+  if len(ret) == 0:
+    logging.info("    No failed tests in testReport, check " +
+        jobConsoleOutput + " for why it was reported failed.")
+  return ret
+
+""" Iterate runs of specfied job within num_prev_days and collect results """
+def find_flaky_tests(jenkins_url, job_name, num_prev_days):
+  global numRunsToExamine
+  all_failing = dict()
+  # First list all builds
+  builds = list_builds(jenkins_url, job_name)
+
+  # Select only those in the last N days
+  min_time = int(time.time()) - SECONDS_PER_DAY * num_prev_days
+  builds = [b for b in builds if (int(b['timestamp']) / 1000) > min_time]
+
+  # Filter out only those that failed
+  failing_build_urls = [(b['url'] , b['timestamp']) for b in builds
+      if (b['result'] in ('UNSTABLE', 'FAILURE'))]
+
+  tnum = len(builds)
+  num = len(failing_build_urls)
+  numRunsToExamine = tnum
+  logging.info("    THERE ARE " + str(num) + " builds (out of " + str(tnum)
+      + ") that have failed tests in the past " + str(num_prev_days) + " days"
+      + ((".", ", as listed below:\n")[num > 0]))
+
+  for failed_build_with_time in failing_build_urls:
+    failed_build = failed_build_with_time[0];
+    jobConsoleOutput = failed_build + "Console";
+    testReport = failed_build + "testReport";
+    testReportApiJson = testReport + "/api/json";
+
+    ts = float(failed_build_with_time[1]) / 1000.
+    st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
+    logging.info("===>%s" % str(testReport) + " (" + st + ")")
+    failing = find_failing_tests(testReportApiJson, jobConsoleOutput)
+    if failing:
+      for ftest in failing:
+        logging.info("    Failed test: %s" % ftest)
+        all_failing[ftest] = all_failing.get(ftest,0)+1
+
+  return all_failing
+
+def main():
+  global numRunsToExamine
+  logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
+  # set up logger to write to stdout
+  soh = logging.StreamHandler(sys.stdout)
+  soh.setLevel(logging.INFO)
+  logger = logging.getLogger()
+  logger.removeHandler(logger.handlers[0])
+  logger.addHandler(soh)
+
+  opts = parse_args()
+  logging.info("****Recently FAILED builds in url: " + opts.jenkins_url
+      + "/job/" + opts.job_name + "")
+
+  all_failing = find_flaky_tests(opts.jenkins_url, opts.job_name,
+      opts.num_prev_days)
+  if len(all_failing) == 0:
+    raise SystemExit(0)
+  logging.info("\nAmong " + str(numRunsToExamine) + " runs examined, all failed "
+      + "tests <#failedRuns: testName>:")
+
+  # print summary section: all failed tests sorted by how many times they failed
+  for tn in sorted(all_failing, key=all_failing.get, reverse=True):
+    logging.info("    " + str(all_failing[tn])+ ": " + tn)
+
+if __name__ == "__main__":
+  main()

dev-support/test-patch.sh

@@ -857,74 +857,6 @@ findModules () {
   rm $TMP_MODULES
   echo $CHANGED_MODULES
 }
-###############################################################################
-### Run the test-contrib target
-runContribTests () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Running contrib tests."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then
-    echo "No contrib tests in this project."
-    return 0
-  fi
-
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
-  echo "NOP"
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 contrib tests{color}.  The patch failed contrib unit tests."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 contrib tests{color}.  The patch passed contrib unit tests."
-  return 0
-}
-
-###############################################################################
-### Run the inject-system-faults target
-checkInjectSystemFaults () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Checking the integrity of system test framework code."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
-  echo "NOP"
-  return 0
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 system test framework{color}.  The patch failed system test framework compile."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 system test framework{color}.  The patch passed system test framework compile."
-  return 0
-}
-
 ###############################################################################
 ### Submit a comment to the defect's Jira
 submitJiraComment () {
@@ -1059,11 +991,7 @@ checkReleaseAuditWarnings
 if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
-  runContribTests
-  (( RESULT = RESULT + $? ))
 fi
-checkInjectSystemFaults
-(( RESULT = RESULT + $? ))
 JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
 $JIRA_COMMENT_FOOTER"
 

hadoop-assemblies/pom.xml

@@ -45,10 +45,10 @@
         <configuration>
           <rules>
             <requireMavenVersion>
-              <version>[3.0.0,)</version>
+              <version>${enforced.maven.version}</version>
             </requireMavenVersion>
             <requireJavaVersion>
-              <version>1.6</version>
+              <version>${enforced.java.version}</version>
             </requireJavaVersion>
           </rules>
         </configuration>

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -48,6 +48,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/shellprofile.d</directory>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>${basedir}/src/main/bin</directory>
       <outputDirectory>/sbin</outputDirectory>

hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml

@@ -41,6 +41,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>shellprofile.d</directory>
+      <outputDirectory>libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>bin</directory>
       <outputDirectory>sbin</outputDirectory>

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -46,6 +46,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/shellprofile.d</directory>
+      <outputDirectory>libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>hadoop-yarn/bin</directory>
       <outputDirectory>sbin</outputDirectory>

hadoop-client/pom.xml

@@ -13,7 +13,7 @@
  limitations under the License. See accompanying LICENSE file.
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 <parent>
    <groupId>org.apache.hadoop</groupId>