Skip to content

Commit 771defb

Browse files
author
Maxime Gréau
authored
Build: Add 3rd party dependencies report generation (#27727)
* Adds task dependenciesInfo to BuildPlugin to generate a CSV file with dependencies information (name,version,url,license) * Adds `ConcatFilesTask.groovy` to concatenates multiple files into one * Adds task `:distribution:generateDependenciesReport` to concatenate `dependencies.csv` files into a single file (`es-dependencies.csv` by default) # Examples: $ gradle dependenciesInfo :distribution:generateDependenciesReport ## Use `csv` system property to customize the output file path $ gradle dependenciesInfo :distribution:generateDependenciesReport -Dcsv=/tmp/elasticsearch-dependencies.csv ## When branch is not master, use `build.branch` system property to generate correct licenses URLs $ gradle dependenciesInfo :distribution:generateDependenciesReport -Dbuild.branch=6.x -Dcsv=/tmp/elasticsearch-dependencies.csv
1 parent 0b2c8c8 commit 771defb

File tree

11 files changed

+270
-0
lines changed

11 files changed

+270
-0
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ forbiddenApis {
6868

6969
// No licenses for our benchmark deps (we don't ship benchmarks)
7070
dependencyLicenses.enabled = false
71+
dependenciesInfo.enabled = false
7172

7273
thirdPartyAudit.excludes = [
7374
// these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)

‎buildSrc/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ if (project != rootProject) {
140140

141141
// build-tools is not ready for primetime with these...
142142
dependencyLicenses.enabled = false
143+
dependenciesInfo.enabled = false
143144
forbiddenApisMain.enabled = false
144145
forbiddenApisTest.enabled = false
145146
jarHell.enabled = false

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ class BuildPlugin implements Plugin<Project> {
8585

8686
configureTest(project)
8787
configurePrecommit(project)
88+
configureDependenciesInfo(project)
8889
}
8990

9091
/** Performs checks on the build environment and prints information about the build environment. */
@@ -637,4 +638,9 @@ class BuildPlugin implements Plugin<Project> {
637638
project.test.mustRunAfter(precommit)
638639
project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided
639640
}
641+
642+
private static configureDependenciesInfo(Project project) {
643+
Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
644+
deps.dependencies = project.configurations.compile.allDependencies
645+
}
640646
}

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/ConcatFilesTask.groovy‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.gradle
21+
22+
import org.gradle.api.DefaultTask
23+
import org.gradle.api.file.FileTree
24+
import org.gradle.api.tasks.Input
25+
import org.gradle.api.tasks.InputFiles
26+
import org.gradle.api.tasks.Optional
27+
import org.gradle.api.tasks.OutputFile
28+
import org.gradle.api.tasks.TaskAction
29+
30+
/**
31+
* Concatenates a list of files into one and removes duplicate lines.
32+
*/
33+
public class ConcatFilesTask extends DefaultTask {
34+
35+
/** List of files to concatenate */
36+
@InputFiles
37+
FileTree files
38+
39+
/** line to add at the top of the target file */
40+
@Input
41+
@Optional
42+
String headerLine
43+
44+
@OutputFile
45+
File target
46+
47+
public ConcatFilesTask() {
48+
description = 'Concat a list of files into one.'
49+
}
50+
51+
@TaskAction
52+
public void concatFiles() {
53+
final StringBuilder output = new StringBuilder()
54+
55+
if (headerLine) {
56+
output.append(headerLine).append('\n')
57+
}
58+
59+
final StringBuilder sb = new StringBuilder()
60+
files.each { file ->
61+
sb.append(file.getText('UTF-8'))
62+
}
63+
// Remove duplicate lines
64+
sb.readLines().toSet().each { value ->
65+
output.append(value).append('\n')
66+
}
67+
68+
target.setText(output.toString(), 'UTF-8')
69+
}
70+
}

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.gradle
21+
22+
import org.gradle.api.DefaultTask
23+
import org.gradle.api.artifacts.Dependency
24+
import org.gradle.api.artifacts.DependencySet
25+
import org.gradle.api.tasks.Input
26+
import org.gradle.api.tasks.InputDirectory
27+
import org.gradle.api.tasks.OutputFile
28+
import org.gradle.api.tasks.TaskAction
29+
30+
31+
/**
32+
* A task to gather information about the dependencies and export them into a csv file.
33+
*
34+
* The following information is gathered:
35+
* <ul>
36+
* <li>name: name that identifies the library (groupId:artifactId)</li>
37+
* <li>version</li>
38+
* <li>URL: link to have more information about the dependency.</li>
39+
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
40+
* </ul>
41+
*
42+
*/
43+
public class DependenciesInfoTask extends DefaultTask {
44+
45+
/** Dependencies to gather information from. */
46+
@Input
47+
public DependencySet dependencies
48+
49+
/** Directory to read license files */
50+
@InputDirectory
51+
public File licensesDir = new File(project.projectDir, 'licenses')
52+
53+
@OutputFile
54+
File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv")
55+
56+
public DependenciesInfoTask() {
57+
description = 'Create a CSV file with dependencies information.'
58+
}
59+
60+
@TaskAction
61+
public void generateDependenciesInfo() {
62+
final StringBuilder output = new StringBuilder()
63+
64+
for (Dependency dependency : dependencies) {
65+
// Only external dependencies are checked
66+
if (dependency.group != null && dependency.group.contains("elasticsearch") == false) {
67+
final String url = createURL(dependency.group, dependency.name, dependency.version)
68+
final String licenseType = getLicenseType(dependency.group, dependency.name)
69+
output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
70+
}
71+
}
72+
outputFile.setText(output.toString(), 'UTF-8')
73+
}
74+
75+
/**
76+
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
77+
* based on dependency coordinates.
78+
*/
79+
protected String createURL(final String group, final String name, final String version){
80+
final String baseURL = 'https://repo1.maven.org/maven2'
81+
return "${baseURL}/${group.replaceAll('\\.' , '/')}/${name}/${version}"
82+
}
83+
84+
/**
85+
* Read the LICENSE file associated with the dependency and determine a license type.
86+
*
87+
* The license type is one of the following values:
88+
* <u>
89+
* <li><em>UNKNOWN</em> if LICENSE file is not present for this dependency.</li>
90+
* <li><em>one SPDX identifier</em> if the LICENSE content matches with an SPDX license.</li>
91+
* <li><em>Custom:URL</em> if it's not an SPDX license,
92+
* URL is the Github URL to the LICENSE file in elasticsearch repository.</li>
93+
* </ul>
94+
*
95+
* @param group dependency group
96+
* @param name dependency name
97+
* @return SPDX identifier, UNKNOWN or a Custom license
98+
*/
99+
protected String getLicenseType(final String group, final String name) {
100+
File license
101+
102+
if (licensesDir.exists()) {
103+
licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file ->
104+
String prefix = file.name.split('-LICENSE.*')[0]
105+
if (group.contains(prefix) || name.contains(prefix)) {
106+
license = file.getAbsoluteFile()
107+
}
108+
}
109+
}
110+
111+
if (license) {
112+
final String content = license.readLines("UTF-8").toString()
113+
final String spdx = checkSPDXLicense(content)
114+
if (spdx == null) {
115+
// License has not be identified as SPDX.
116+
// As we have the license file, we create a Custom entry with the URL to this license file.
117+
final gitBranch = System.getProperty('build.branch', 'master')
118+
final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/"
119+
return "Custom:${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}"
120+
}
121+
return spdx
122+
} else {
123+
return "UNKNOWN"
124+
}
125+
}
126+
127+
/**
128+
* Check the license content to identify an SPDX license type.
129+
*
130+
* @param licenseText LICENSE file content.
131+
* @return SPDX identifier or null.
132+
*/
133+
private String checkSPDXLicense(final String licenseText) {
134+
String spdx = null
135+
136+
final String APACHE_2_0 = "Apache.*License.*(v|V)ersion 2.0"
137+
final String BSD_2 = "BSD 2-clause.*License"
138+
final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0"
139+
final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1"
140+
final String ICU = "ICU License - ICU 1.8.1 and later"
141+
final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3"
142+
final String MIT = "MIT License"
143+
final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1"
144+
145+
switch (licenseText) {
146+
case ~/.*${APACHE_2_0}.*/:
147+
spdx = 'Apache-2.0'
148+
break
149+
case ~/.*${MIT}.*/:
150+
spdx = 'MIT'
151+
break
152+
case ~/.*${BSD_2}.*/:
153+
spdx = 'BSD-2-Clause'
154+
break
155+
case ~/.*${LGPL_3}.*/:
156+
spdx = 'LGPL-3.0'
157+
break
158+
case ~/.*${CDDL_1_0}.*/:
159+
spdx = 'CDDL_1_0'
160+
break
161+
case ~/.*${CDDL_1_1}.*/:
162+
spdx = 'CDDL_1_1'
163+
break
164+
case ~/.*${ICU}.*/:
165+
spdx = 'ICU'
166+
break
167+
case ~/.*${MOZILLA_1_1}.*/:
168+
spdx = 'MPL-1.1'
169+
break
170+
default:
171+
break
172+
}
173+
return spdx
174+
}
175+
}

‎client/benchmark/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,4 @@ dependencies {
6767

6868
// No licenses for our benchmark deps (we don't ship benchmarks)
6969
dependencyLicenses.enabled = false
70+
dependenciesInfo.enabled = false

‎client/test/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ jarHell.enabled=false
5353

5454
// TODO: should we have licenses for our test deps?
5555
dependencyLicenses.enabled = false
56+
dependenciesInfo.enabled = false
5657

5758
namingConventions.enabled = false
5859

‎distribution/build.gradle‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.apache.tools.ant.filters.FixCrLfFilter
2222
import org.apache.tools.ant.taskdefs.condition.Os
2323
import org.elasticsearch.gradle.BuildPlugin
2424
import org.elasticsearch.gradle.EmptyDirTask
25+
import org.elasticsearch.gradle.ConcatFilesTask
2526
import org.elasticsearch.gradle.MavenFilteringHack
2627
import org.elasticsearch.gradle.NoticeTask
2728
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
@@ -43,6 +44,17 @@ buildscript {
4344
Collection distributions = project.subprojects.findAll {
4445
it.path.contains(':tools') == false && it.path.contains(':bwc') == false }
4546

47+
/*****************************************************************************
48+
* Third party dependencies report *
49+
*****************************************************************************/
50+
51+
// Concatenates the dependencies CSV files into a single file
52+
task generateDependenciesReport(type: ConcatFilesTask) {
53+
files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' )
54+
headerLine = "name,version,url,license"
55+
target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv")
56+
}
57+
4658
/*****************************************************************************
4759
* Notice file *
4860
*****************************************************************************/

‎qa/wildfly/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,5 +193,6 @@ check.dependsOn(integTest)
193193
test.enabled = false
194194

195195
dependencyLicenses.enabled = false
196+
dependenciesInfo.enabled = false
196197

197198
thirdPartyAudit.enabled = false

‎test/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ subprojects {
3838

3939
// TODO: should we have licenses for our test deps?
4040
dependencyLicenses.enabled = false
41+
dependenciesInfo.enabled = false
4142

4243
// TODO: why is the test framework pulled in...
4344
forbiddenApisMain.enabled = false

0 commit comments

Comments
 (0)