diff --git a/LICENSE-binary b/LICENSE-binary index 3b8076e6de141..1661ed8eb6bdb 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -297,7 +297,6 @@ io.swagger:swagger-annotations:1.5.4 javax.inject:javax.inject:1 net.java.dev.jna:jna:5.2.0 net.minidev:accessors-smart:1.2 -org.apache.avro:avro:1.9.2 org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.24.0 org.apache.commons:commons-configuration2:2.8.0 diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 666df82f48381..64021f033508c 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -136,6 +136,10 @@ org.apache.avro avro + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + org.apache.curator curator-client diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index 7e6ef037ee2d7..b845819de3e77 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -114,6 +114,10 @@ org.apache.avro avro + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + org.eclipse.jetty jetty-server @@ -166,6 +170,10 @@ org.apache.avro avro + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + com.sun.jersey jersey-core @@ -278,6 +286,10 @@ org.apache.avro avro + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + org.apache.hadoop hadoop-annotations @@ -314,6 +326,10 @@ org.apache.avro avro + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + org.apache.hadoop hadoop-annotations diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 3b03fc6aadb7e..7e3640c268729 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -249,8 +249,8 @@ test - org.apache.avro - avro + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 compile @@ -617,6 +617,8 @@ + + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java index 155381de949ef..18fcd1d2320b9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java @@ -21,7 +21,7 @@ import java.io.Closeable; import java.io.IOException; -import org.apache.avro.file.SeekableInput; +import org.apache.hadoop.thirdparty.avro.file.SeekableInput; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index f4dec29a640b4..8576b7dd7fc32 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -27,7 +27,7 @@ import java.util.Optional; import java.util.regex.Pattern; -import org.apache.avro.reflect.Stringable; +import org.apache.hadoop.thirdparty.avro.reflect.Stringable; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java index 1a3f18a658d22..5914e8079195a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java @@ -25,14 +25,14 @@ import java.util.LinkedList; import java.util.zip.GZIPInputStream; -import org.apache.avro.Schema; -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.FileReader; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.io.JsonEncoder; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.file.DataFileReader; +import org.apache.hadoop.thirdparty.avro.file.FileReader; +import org.apache.hadoop.thirdparty.avro.generic.GenericDatumReader; +import org.apache.hadoop.thirdparty.avro.generic.GenericDatumWriter; +import org.apache.hadoop.thirdparty.avro.io.DatumWriter; +import org.apache.hadoop.thirdparty.avro.io.EncoderFactory; +import org.apache.hadoop.thirdparty.avro.io.JsonEncoder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index 86fb1ff9a54a2..2e715cc1b1d54 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -33,7 +33,7 @@ import java.text.StringCharacterIterator; import java.util.Arrays; -import org.apache.avro.reflect.Stringable; +import org.apache.hadoop.thirdparty.avro.reflect.Stringable; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java index 544958e682a50..4d26242f9fc29 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java @@ -21,12 +21,12 @@ import java.util.HashSet; import java.util.Set; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.DatumReader; +import org.apache.hadoop.thirdparty.avro.io.DatumWriter; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectData; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java index 2327fd2d55a2e..7585b7b9481d0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java @@ -22,13 +22,13 @@ import java.io.InputStream; import java.io.OutputStream; -import org.apache.avro.Schema; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.BinaryDecoder; +import org.apache.hadoop.thirdparty.avro.io.BinaryEncoder; +import org.apache.hadoop.thirdparty.avro.io.DatumReader; +import org.apache.hadoop.thirdparty.avro.io.DatumWriter; +import org.apache.hadoop.thirdparty.avro.io.DecoderFactory; +import org.apache.hadoop.thirdparty.avro.io.EncoderFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configured; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java index e49d7a09dd53a..562c5ab96bc39 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java @@ -18,12 +18,12 @@ package org.apache.hadoop.io.serializer.avro; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecord; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.DatumReader; +import org.apache.hadoop.thirdparty.avro.io.DatumWriter; +import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader; +import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter; +import org.apache.hadoop.thirdparty.avro.specific.SpecificRecord; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java index 1a4e7db30415a..a12c733142b45 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java @@ -25,7 +25,7 @@ import javax.crypto.SecretKey; -import org.apache.avro.reflect.Nullable; +import org.apache.hadoop.thirdparty.avro.reflect.Nullable; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java index 9c9b75fa76e6c..d01e0a94873c4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java @@ -21,12 +21,12 @@ import java.io.ByteArrayOutputStream; import java.lang.reflect.Type; -import org.apache.avro.Schema; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.io.DecoderFactory; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.EncoderFactory; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectData; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter; +import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader; +import org.apache.hadoop.thirdparty.avro.io.DecoderFactory; import static org.junit.Assert.assertEquals; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java index b2d2a8d100ff1..97e281ba85ea2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java @@ -33,7 +33,7 @@ public class TestAvroSerialization { @Test public void testSpecific() throws Exception { AvroRecord before = new AvroRecord(); - before.intField = 5; + before.setIntField(5); AvroRecord after = SerializationTestUtil.testSerialization(conf, before); assertEquals(before, after); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index e43e2ca24e92c..915850f14d60b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -131,6 +131,19 @@ maven-antrun-plugin + generate-sources-ant-replace + generate-sources + + run + + + + + + + + + pre-site-ant-copy pre-site run diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java index a3fa57ee242b7..3ab48cb510f58 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java index 608756657d55c..d9e7a70750ab1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java @@ -22,8 +22,8 @@ import java.util.Iterator; import java.util.List; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.generic.GenericData; public class AvroArrayUtils { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java index 3c7055ce33a60..0c4e00258bd22 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java @@ -23,13 +23,13 @@ import java.io.EOFException; import java.io.IOException; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.Decoder; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificDatumReader; +import org.apache.hadoop.thirdparty.avro.AvroRuntimeException; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.DatumReader; +import org.apache.hadoop.thirdparty.avro.io.Decoder; +import org.apache.hadoop.thirdparty.avro.io.DecoderFactory; +import org.apache.hadoop.thirdparty.avro.specific.SpecificData; +import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java index 3ab3d024e7bfa..9a7e9963185a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.Schema; +import org.apache.hadoop.thirdparty.avro.io.DatumWriter; +import org.apache.hadoop.thirdparty.avro.io.Encoder; +import org.apache.hadoop.thirdparty.avro.io.EncoderFactory; +import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataOutputStream; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java index 34e4b2c89c8f4..493bdb909dafc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.Counters; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java index f5941aa04479d..639770a318909 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.JobID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java index 784267f2b0a70..efa61403145e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.JobID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java index 1616dd1962b2b..42b2bf6b0f054 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.JobPriority; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java index 66f378123986d..4ac9f7a294ea4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java @@ -20,8 +20,8 @@ import java.util.Set; -import org.apache.avro.util.Utf8; import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; @@ -31,8 +31,8 @@ public class JobQueueChangeEvent implements HistoryEvent { private JobQueueChange datum = new JobQueueChange(); public JobQueueChangeEvent(JobID id, String queueName) { - datum.jobid = new Utf8(id.toString()); - datum.jobQueueName = new Utf8(queueName); + datum.setJobid(new Utf8(id.toString())); + datum.setJobQueueName(new Utf8(queueName)); } JobQueueChangeEvent() { } @@ -54,13 +54,13 @@ public void setDatum(Object datum) { /** Get the Job ID */ public JobID getJobId() { - return JobID.forName(datum.jobid.toString()); + return JobID.forName(datum.getJobid().toString()); } /** Get the new Job queue name */ public String getJobQueueName() { - if (datum.jobQueueName != null) { - return datum.jobQueueName.toString(); + if (datum.getJobQueueName() != null) { + return datum.getJobQueueName().toString(); } return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java index 0963b45b492bd..302072b1a1c20 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.JobID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java index e394f5bbf6ec2..4564beaa1d79c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java @@ -23,7 +23,7 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.JobConf; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java index 4c43c7e0c2613..0b54d03000b54 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java @@ -21,7 +21,7 @@ import java.util.Collections; import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.JobID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index 2b1357ea85902..d80d0cd28590b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.ProgressSplitsBlock; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index 5a16f834acb4a..916b01b7bc3a2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.ProgressSplitsBlock; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java index c28c21605df86..2c9c46f0a3e36 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.Counters; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java index 275ff1d187122..dc98e91650eb8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.TaskAttemptID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index 9afa09384cc9e..4154cc73eae49 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.ProgressSplitsBlock; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java index b4d9e410da27d..ec796a36d1cab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.TaskStatus; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index 97557c7e0b4fe..eb4588926292d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.TaskStatus; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java index 9d2fc0eee5541..ff829d6ec4d95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.TaskID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java index 010129d73ceb4..258e7929c13b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java @@ -20,7 +20,7 @@ import java.util.Set; -import org.apache.avro.util.Utf8; +import org.apache.hadoop.thirdparty.avro.util.Utf8; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapreduce.TaskID; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java index 64cb253d9c587..1bd563e822027 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java @@ -25,7 +25,7 @@ import org.junit.Assert; -import org.apache.avro.AvroRemoteException; +import org.apache.hadoop.thirdparty.avro.AvroRemoteException; import org.apache.hadoop.mapreduce.SleepJob; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index 3f7b0d4f88fa9..4456d047a6791 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -41,8 +41,18 @@ hadoop-mapreduce-client-core - org.apache.avro - avro + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + + + org.slf4j + slf4j-api + + + org.xerial.snappy + snappy-java + + test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 5bbb786a016e9..e7e4525cecbaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -42,33 +42,13 @@ ${transient.protobuf2.scope} - org.apache.avro - avro + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 - - org.eclipse.jetty - jetty-server - - - org.apache.ant - ant - - - org.jboss.netty - netty - - - org.apache.velocity - velocity - org.slf4j slf4j-api - - paranamer-ant - com.thoughtworks.paranamer - org.xerial.snappy snappy-java diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 968527039bd86..088e12f336463 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -51,6 +51,25 @@ hadoop-mapreduce-client-app ${project.version} + + com.google.protobuf + protobuf-java + + + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + ${hadoop-thirdparty.avro.version} + + + org.slf4j + slf4j-api + + + org.xerial.snappy + snappy-java + + + org.apache.hadoop hadoop-mapreduce-client-common diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 1cf9401d707c3..e86b251efc04b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -62,8 +62,8 @@ ${basedir}/../../hadoop-common-project/hadoop-common/target file:///dev/urandom - - 1.9.2 + + 1.11.3 1.19.4 @@ -96,7 +96,9 @@ 3.23.4 ${env.HADOOP_PROTOC_PATH} + 1.2.0 + ${hadoop-thirdparty.version} 1.3.0-SNAPSHOT ${hadoop-thirdparty.version} org.apache.hadoop.thirdparty @@ -1358,9 +1360,9 @@ - org.apache.avro - avro - ${avro.version} + org.apache.hadoop.thirdparty + hadoop-shaded-avro_1_11 + ${hadoop-thirdparty.avro.version} net.sf.kosmosfs diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java index 1213e6a46f449..dcb6bed166df9 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java @@ -89,7 +89,7 @@ public class JobBuilder { private Map allHosts = new HashMap(); - private org.apache.hadoop.mapreduce.jobhistory.JhCounters EMPTY_COUNTERS = + private final static org.apache.hadoop.mapreduce.jobhistory.JhCounters EMPTY_COUNTERS = new org.apache.hadoop.mapreduce.jobhistory.JhCounters(); /** @@ -460,7 +460,7 @@ private void processTaskFinishedEvent(TaskFinishedEvent event) { } task.setFinishTime(event.getFinishTime()); task.setTaskStatus(getPre21Value(event.getTaskStatus())); - task.incorporateCounters(((TaskFinished) event.getDatum()).counters); + task.incorporateCounters(((TaskFinished) event.getDatum()).getCounters()); } private void processTaskFailedEvent(TaskFailedEvent event) { @@ -472,7 +472,7 @@ private void processTaskFailedEvent(TaskFailedEvent event) { task.setFinishTime(event.getFinishTime()); task.setTaskStatus(getPre21Value(event.getTaskStatus())); TaskFailed t = (TaskFailed)(event.getDatum()); - task.putDiagnosticInfo(t.error.toString()); + task.putDiagnosticInfo(t.getError().toString()); // killed task wouldn't have failed attempt. if (t.getFailedDueToAttempt() != null) { task.putFailedDueToAttemptId(t.getFailedDueToAttempt().toString()); @@ -542,7 +542,7 @@ private void processTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) { } attempt.setFinishTime(event.getFinishTime()); attempt - .incorporateCounters(((TaskAttemptFinished) event.getDatum()).counters); + .incorporateCounters(((TaskAttemptFinished) event.getDatum()).getCounters()); } private void processReduceAttemptFinishedEvent( @@ -568,7 +568,7 @@ private void processReduceAttemptFinishedEvent( attempt.setShuffleFinished(event.getShuffleFinishTime()); attempt.setSortFinished(event.getSortFinishTime()); attempt - .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters); + .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).getCounters()); attempt.arraySetClockSplits(event.getClockSplits()); attempt.arraySetCpuUsages(event.getCpuUsages()); attempt.arraySetVMemKbytes(event.getVMemKbytes()); @@ -596,7 +596,7 @@ private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) { // is redundant, but making this will add future-proofing. attempt.setFinishTime(event.getFinishTime()); attempt - .incorporateCounters(((MapAttemptFinished) event.getDatum()).counters); + .incorporateCounters(((MapAttemptFinished) event.getDatum()).getCounters()); attempt.arraySetClockSplits(event.getClockSplits()); attempt.arraySetCpuUsages(event.getCpuUsages()); attempt.arraySetVMemKbytes(event.getVMemKbytes()); @@ -661,11 +661,11 @@ private void processJobFinishedEvent(JobFinishedEvent event) { JobFinished job = (JobFinished)event.getDatum(); Map countersMap = - JobHistoryUtils.extractCounters(job.totalCounters); + JobHistoryUtils.extractCounters(job.getTotalCounters()); result.putTotalCounters(countersMap); - countersMap = JobHistoryUtils.extractCounters(job.mapCounters); + countersMap = JobHistoryUtils.extractCounters(job.getMapCounters()); result.putMapCounters(countersMap); - countersMap = JobHistoryUtils.extractCounters(job.reduceCounters); + countersMap = JobHistoryUtils.extractCounters(job.getReduceCounters()); result.putReduceCounters(countersMap); } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java index 6ae87bbd40a11..34ef95f337858 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java @@ -157,9 +157,9 @@ static boolean isJobConfXml(String fileName) { static Map extractCounters(JhCounters counters) { Map countersMap = new HashMap(); if (counters != null) { - for (JhCounterGroup group : counters.groups) { - for (JhCounter counter : group.counts) { - countersMap.put(counter.name.toString(), counter.value); + for (JhCounterGroup group : counters.getGroups()) { + for (JhCounter counter : group.getCounts()) { + countersMap.put(counter.getName().toString(), counter.getValue()); } } } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java index 4ae33a76617fb..2308e586900bb 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java @@ -268,11 +268,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters, String counterName) { counterName = canonicalizeCounterName(counterName); - for (JhCounterGroup group : counters.groups) { - for (JhCounter counter : group.counts) { + for (JhCounterGroup group : counters.getGroups()) { + for (JhCounter counter : group.getCounts()) { if (counterName - .equals(canonicalizeCounterName(counter.name.toString()))) { - thunk.set(counter.value); + .equals(canonicalizeCounterName(counter.getName().toString()))) { + thunk.set(counter.getValue()); return; } } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java index 5c6abd372c081..c4ca962b6e69f 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java @@ -636,11 +636,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters, String counterName) { counterName = canonicalizeCounterName(counterName); - for (JhCounterGroup group : counters.groups) { - for (JhCounter counter : group.counts) { + for (JhCounterGroup group : counters.getGroups()) { + for (JhCounter counter : group.getCounts()) { if (counterName - .equals(canonicalizeCounterName(counter.name.toString()))) { - thunk.set(counter.value); + .equals(canonicalizeCounterName(counter.getName().toString()))) { + thunk.set(counter.getValue()); return; } }