Skip to content

Commit fccf35a

Browse files
turboFeiWu, Jieping
authored and
GitHub Enterprise
committed
[HADP-54780] Setup ADLC release for etl spark (apache#531)
<img width="1328" alt="image" src="https://media.github.corp.ebay.com/user/27990/files/975c93e5-3fe8-4712-9a8a-bcb6fef318b8"> --------- Co-authored-by: Wu, Jieping <[email protected]>
1 parent 43988a7 commit fccf35a

File tree

4 files changed

+327
-0
lines changed

4 files changed

+327
-0
lines changed

‎Dockerfile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
FROM hub.tess.io/adihadoop/maven:3.8-jdk-8-slim AS builder
19+
20+
RUN apt update && apt -y --allow-downgrades install tar zip git
21+
22+
RUN /workspace/build/adlc/release.sh --adlc-cleanup
23+
24+
FROM scratch
25+
COPY --from=builder /workspace/*.tgz ./
26+
COPY --from=builder /workspace/*.zip ./
27+
COPY --from=builder /workspace/*.md5 ./
28+
COPY --from=builder /workspace/*.RELEASE ./
29+
30+
LABEL com.ebay.adi.adlc.include="*.tgz,*.zip,*.md5,*.RELEASE"
31+
# please update the version when releaseing new Spark binary
32+
LABEL com.ebay.adi.adlc.tag="3.5.0-ebay.0-SNAPSHOT"
33+
# public for read
34+
LABEL com.ebay.adi.adlc.viewer="*"

‎build/adlc/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Spark ADLC
2+
3+
- ADLC application address: https://cloud.ebay.com/object/detail/Application:spark3adlc
4+
- Pipeline address: https://cloud.ebay.com/object/detail/Application:spark3adlc/Pipeline
5+
- Image repo: https://hub.tess.io/repository/adihadoop/spark3adlc
6+
- Hdfs folder for latest spark binaries: /apps/b_adlc/b_hive_dba/spark3adlc/${SPARK_VERSION}/latest
7+
- Please update the `com.ebay.adi.adlc.tag` in [Dockerfile](../../Dockerfile) when release new Spark version.

‎build/adlc/maven-settings.xml‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<settings>
3+
<localRepository>${user.home}/.m2/raptor2</localRepository>
4+
<!--
5+
For Authenticated accss one server entry needed per ID in
6+
Profiles -> Profile -> repositories ->
7+
-->
8+
<servers>
9+
<server>
10+
<id>raptor.aggregator</id>
11+
<username>RaptorTeam</username>
12+
<password>RaptorTeam</password>
13+
</server>
14+
<server>
15+
<id>central</id>
16+
<username>RaptorTeam</username>
17+
<password>RaptorTeam</password>
18+
</server>
19+
<server>
20+
<id>raptor.releases</id>
21+
<username>RaptorTeam</username>
22+
<password>RaptorTeam</password>
23+
</server>
24+
<server>
25+
<id>raptor.snapshots</id>
26+
<username>RaptorTeam</username>
27+
<password>RaptorTeam</password>
28+
</server>
29+
<server>
30+
<id>raptor.plugins</id>
31+
<username>RaptorTeam</username>
32+
<password>RaptorTeam</password>
33+
</server>
34+
<server>
35+
<id>raptor.thirdparty</id>
36+
<username>RaptorTeam</username>
37+
<password>RaptorTeam</password>
38+
</server>
39+
<server>
40+
<id>raptor.commercial</id>
41+
<username>RaptorTeam</username>
42+
<password>RaptorTeam</password>
43+
</server>
44+
<server>
45+
<id>raptor.v3debt</id>
46+
<username>RaptorTeam</username>
47+
<password>RaptorTeam</password>
48+
</server>
49+
</servers>
50+
<mirrors>
51+
<mirror>
52+
<id>central</id>
53+
<!--
54+
Below entry Honors all repositories defined in the pom file of direct/transitive dependencies
55+
-->
56+
<mirrorOf>central</mirrorOf>
57+
<!--
58+
Following entry forces maven to use only ebaycentral.qa.ebay.com. This improved build time.
59+
If you add new repository, make sure to enter the text !newrepoID in the list below, and add auth above if that repo needs authentication.
60+
-->
61+
<!-- <mirrorOf>*,!raptor.aggregator,!raptor.releases,!raptor.snapshots,!raptor.thirdparty,!raptor.plugins,!raptor.commercial,!raptor.v3debt,!raptor.extn1,!raptor.extn2</mirrorOf> -->
62+
<url>https://ebaycentral.qa.ebay.com/content/repositories/central</url>
63+
</mirror>
64+
</mirrors>
65+
<profiles>
66+
<profile>
67+
<id>ebay</id>
68+
<!-- add a repository property to ease the archetype:generate usage -->
69+
<properties>
70+
<archetypeCatalog>https://ebaycentral.qa.ebay.com/content/repositories/releases/archetype-catalog.xml</archetypeCatalog>
71+
</properties>
72+
<repositories>
73+
<!-- Raptor Repository -->
74+
<repository>
75+
<id>raptor.aggregator</id>
76+
<url>https://ebaycentral.corp.ebay.com/content/groups/aggregator/</url>
77+
<releases>
78+
<enabled>true</enabled>
79+
<updatePolicy>never</updatePolicy>
80+
</releases>
81+
<snapshots>
82+
<enabled>false</enabled>
83+
</snapshots>
84+
</repository>
85+
<repository>
86+
<id>raptor.releases</id>
87+
<url>https://ebaycentral.qa.ebay.com/content/repositories/releases</url>
88+
<releases>
89+
<enabled>true</enabled>
90+
</releases>
91+
<snapshots>
92+
<enabled>false</enabled>
93+
</snapshots>
94+
</repository>
95+
<repository>
96+
<id>raptor.v3debt</id>
97+
<url>https://ebaycentral.qa.ebay.com/content/repositories/v3debt</url>
98+
<releases>
99+
<enabled>true</enabled>
100+
</releases>
101+
<snapshots>
102+
<enabled>false</enabled>
103+
</snapshots>
104+
</repository>
105+
<repository>
106+
<id>raptor.snapshots</id>
107+
<url>https://ebaycentral.qa.ebay.com/content/repositories/snapshots</url>
108+
<releases>
109+
<enabled>false</enabled>
110+
</releases>
111+
<snapshots>
112+
<enabled>true</enabled>
113+
</snapshots>
114+
</repository>
115+
<repository>
116+
<id>raptor.thirdparty</id>
117+
<url>https://ebaycentral.qa.ebay.com/content/repositories/thirdparty</url>
118+
<releases>
119+
<enabled>true</enabled>
120+
</releases>
121+
<snapshots>
122+
<enabled>false</enabled>
123+
</snapshots>
124+
</repository>
125+
<repository>
126+
<id>raptor.commercial</id>
127+
<url>https://ebaycentral.qa.ebay.com/content/repositories/commercial</url>
128+
<releases>
129+
<enabled>true</enabled>
130+
</releases>
131+
<snapshots>
132+
<enabled>false</enabled>
133+
</snapshots>
134+
</repository>
135+
</repositories>
136+
<pluginRepositories>
137+
<pluginRepository>
138+
<id>raptor.aggregator</id>
139+
<url>https://ebaycentral.corp.ebay.com/content/groups/aggregator/</url>
140+
<releases>
141+
<enabled>true</enabled>
142+
<updatePolicy>never</updatePolicy>
143+
</releases>
144+
<snapshots>
145+
<enabled>false</enabled>
146+
</snapshots>
147+
</pluginRepository>
148+
<pluginRepository>
149+
<id>central</id>
150+
<url>https://ebaycentral.qa.ebay.com/content/repositories/central</url>
151+
<releases>
152+
<enabled>true</enabled>
153+
</releases>
154+
<snapshots>
155+
<enabled>false</enabled>
156+
</snapshots>
157+
</pluginRepository>
158+
<pluginRepository>
159+
<id>raptor.releases</id>
160+
<url>https://ebaycentral.qa.ebay.com/content/repositories/releases</url>
161+
<releases>
162+
<enabled>true</enabled>
163+
</releases>
164+
<snapshots>
165+
<enabled>false</enabled>
166+
</snapshots>
167+
</pluginRepository>
168+
<pluginRepository>
169+
<id>raptor.v3debt</id>
170+
<url>https://ebaycentral.qa.ebay.com/content/repositories/v3debt</url>
171+
<releases>
172+
<enabled>true</enabled>
173+
</releases>
174+
<snapshots>
175+
<enabled>false</enabled>
176+
</snapshots>
177+
</pluginRepository>
178+
<pluginRepository>
179+
<id>raptor.plugins</id>
180+
<url>https://ebaycentral.qa.ebay.com/content/repositories/plugins</url>
181+
<releases>
182+
<enabled>true</enabled>
183+
</releases>
184+
<snapshots>
185+
<enabled>false</enabled>
186+
</snapshots>
187+
</pluginRepository>
188+
<pluginRepository>
189+
<id>raptor.snapshots</id>
190+
<url>https://ebaycentral.qa.ebay.com/content/repositories/snapshots</url>
191+
<releases>
192+
<enabled>false</enabled>
193+
</releases>
194+
<snapshots>
195+
<enabled>true</enabled>
196+
</snapshots>
197+
</pluginRepository>
198+
<pluginRepository>
199+
<id>raptor.thirdparty</id>
200+
<url>https://ebaycentral.qa.ebay.com/content/repositories/thirdparty</url>
201+
<releases>
202+
<enabled>true</enabled>
203+
</releases>
204+
<snapshots>
205+
<enabled>false</enabled>
206+
</snapshots>
207+
</pluginRepository>
208+
</pluginRepositories>
209+
</profile>
210+
</profiles>
211+
<activeProfiles>
212+
<activeProfile>ebay</activeProfile>
213+
</activeProfiles>
214+
<pluginGroups>
215+
<pluginGroup>org.mortbay.jetty</pluginGroup>
216+
<pluginGroup>com.ebay.raptor.build</pluginGroup>
217+
<pluginGroup>org.codehaus.mojo</pluginGroup>
218+
<pluginGroup>org.sonatype.maven.plugins</pluginGroup>
219+
</pluginGroups>
220+
</settings>

‎build/adlc/release.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
set -o pipefail
21+
set -e
22+
23+
SPARK_HOME="$(cd "`dirname "$0"`/../.."; pwd)"
24+
cd $SPARK_HOME
25+
26+
rm -rf "$SPARK_HOME/.git"
27+
28+
export MAVEN_OPTS="-Xmx4g -XX:ReservedCodeCacheSize=512m"
29+
dev/make-distribution.sh --name ebay -Phadoop-3 -Phive -Phive-thriftserver -Pyarn -Pebay-etl-sql -Pkubernetes -U -s build/adlc/maven-settings.xml
30+
31+
VERSION=$(build/mvn help:evaluate -Dexpression=project.version 2>/dev/null| grep -v "INFO"| grep -v "WARNING"| tail -n 1)
32+
33+
ADLC_VERSION=$(grep com.ebay.adi.adlc.tag Dockerfile|awk -F '=' '{print $2}'|tr -d '"')
34+
35+
if [[ $VERSION != $ADLC_VERSION ]]; then
36+
echo "The project version and ADLC version do not match. Please update the ADLC version in Dockerfile."
37+
exit 1
38+
fi
39+
40+
# 1. prepare variables
41+
TARDIR_NAME=spark-$VERSION-bin-ebay
42+
TARDIR="$SPARK_HOME/$TARDIR_NAME"
43+
DISTDIR="$SPARK_HOME/dist"
44+
rm -rf $TARDIR
45+
cp -r $DISTDIR $TARDIR
46+
47+
# 2. delete zookeeper & htrace jar
48+
rm -f $TARDIR/jars/zookeeper-*.jar
49+
rm -f $TARDIR/jars/htrace-*.jar
50+
51+
# 3. prepare spark archive
52+
cd $TARDIR/jars/ && zip spark-$VERSION-jars.zip *.jar && cd -
53+
mv $TARDIR/jars/spark-$VERSION-jars.zip .
54+
55+
# 4. clean ADLC workspace
56+
rm -rf "$SPARK_HOME/**/src/" "$SPARK_HOME/**/target/"
57+
58+
# 5. compress the binary
59+
tar czf "spark-$VERSION-bin-ebay.tgz" -C . $TARDIR_NAME
60+
61+
# 6. md5 and RELEASE
62+
md5sum spark-$VERSION-bin-ebay.tgz > $TARDIR_NAME.md5
63+
md5sum spark-$VERSION-jars.zip >> $TARDIR_NAME.md5
64+
cp $TARDIR/RELEASE $TARDIR_NAME.RELEASE
65+
66+
rm -rf $TARDIR

0 commit comments

Comments
 (0)