Skip to content

Commit ac3198f

Browse files
TravisEz13SeeminglyScience
TravisEz13
authored and
SeeminglyScience
committed
Add tool to trigger license information gathering for NuGet modules (PowerShell#18827)
1 parent db10b20 commit ac3198f

File tree

4 files changed

+216
-0
lines changed

4 files changed

+216
-0
lines changed

‎.spelling

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1589,3 +1589,5 @@ centos-7
15891589
Security.types.ps1xml
15901590
- ADOPTERS.md
15911591
MicrosoftPowerBIMgmt
1592+
- tools/clearlyDefined/readme.md
1593+
ClearlyDefined

‎tools/clearlyDefined/ClearlyDefined.ps1

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
param(
4+
[parameter(Mandatory = $true, ParameterSetName='Harvest')]
5+
[switch]
6+
$Harvest,
7+
[parameter(Mandatory = $true, ParameterSetName='Test')]
8+
[switch]
9+
$Test,
10+
[switch]
11+
$ForceModuleReload
12+
)
13+
14+
$extraParams = @{}
15+
if ($ForceModuleReload) {
16+
$extraParams['Force'] = $true
17+
}
18+
19+
Import-Module -Name "$PSScriptRoot/src/ClearlyDefined" @extraParams
20+
21+
$cgManfest = Get-Content "$PSScriptRoot/../cgmanifest.json" | ConvertFrom-Json
22+
$fullCgList = $cgManfest.Registrations.Component |
23+
ForEach-Object {
24+
[Pscustomobject]@{
25+
type = $_.Type
26+
Name = $_.Nuget.Name
27+
PackageVersion = $_.Nuget.Version
28+
}
29+
}
30+
31+
$fullList = $fullCgList | Get-ClearlyDefinedData
32+
33+
$needHarvest = $fullList | Where-Object { !$_.harvested }
34+
35+
Write-Verbose "Full List count: $($fullList.Count)" -Verbose
36+
Write-Verbose "Need harvest: $($needHarvest.Count)" -Verbose
37+
38+
if ($Harvest) {
39+
$needHarvest | select-object -ExpandProperty coordinates | Start-ClearlyDefinedHarvest
40+
} elseif ($Test) {
41+
if($needHarvest.Count -gt 0) {
42+
$needHarvest | Format-List | Out-String -Width 9999 | Write-Verbose -Verbose
43+
throw "There are $($needHarvest.Count) packages that need to be harvested"
44+
} else {
45+
Write-Verbose "All packages have been harvested" -Verbose
46+
}
47+
}

‎tools/clearlyDefined/readme.md

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# ClearlyDefined
2+
3+
## Purpose
4+
5+
This tool is intended to test if all the license data in [ClearlyDefined](https://clearlydefined.io) is present to generate the PowerShell license.
6+
If the data is not present, it can request that ClearlyDefined gather (called Harvest in their terminology) the data.
7+
8+
## Use
9+
10+
### Testing
11+
12+
Run `./ClearlyDefined.ps1 -test`.
13+
14+
If there is any missing data, the script should write verbose messages about the missing data and throw.
15+
If there is no missing data, the script should not throw.
16+
17+
### Harvesting
18+
19+
Run `./ClearlyDefined.ps1 -Harvest`.
20+
The script will trigger the harvest and output the result from ClearlyDefined.
21+
**Give ClearlyDefined 24 hours to harvest the data.**
22+
You can use the `-Test` switch without the `-Harvest` switch to test if Harvesting is done.
23+
24+
## Caching
25+
26+
If you run in the same PowerShell session, the script will be faster due to caching.
27+
28+
The module will cache any results from ClearlyDefined that indicate the package is Harvested for 60 minutes.
29+
No caching is done for packages that are not yet harvested.
30+
To clear the cache, run with the `-ForceModuleReload` switch.

‎tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
# Start the collection (known as harvest) of ClearlyDefined data for a package
5+
function Start-ClearlyDefinedHarvest {
6+
[CmdletBinding()]
7+
param(
8+
[Parameter(ValueFromPipelineByPropertyName=$true)]
9+
[Alias('Type')]
10+
[validateset('nuget')]
11+
[string]
12+
$PackageType = 'nuget',
13+
14+
[parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
15+
[Alias('Name')]
16+
[string]
17+
$PackageName,
18+
19+
[parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
20+
[Alias('Version')]
21+
[Alias('Revision')]
22+
[string]
23+
$PackageVersion
24+
)
25+
26+
Process {
27+
$coordinates = Get-ClearlyDefinedCoordinates @PSBoundParameters
28+
$body = @{tool='package';coordinates=$coordinates} | convertto-json
29+
Write-Verbose $body -Verbose
30+
(Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json').Content
31+
}
32+
}
33+
34+
function ConvertFrom-ClearlyDefinedCoordinates {
35+
[CmdletBinding()]
36+
param(
37+
[parameter(mandatory = $true, ValueFromPipeline = $true)]
38+
[string]
39+
$Coordinates
40+
)
41+
42+
Begin {}
43+
Process {
44+
$parts = $Coordinates.Split('/')
45+
[PSCustomObject]@{
46+
type = $parts[0]
47+
provider = $parts[1]
48+
namespace = $parts[2]
49+
name = $parts[3]
50+
revision = $parts[4]
51+
}
52+
}
53+
End {}
54+
}
55+
56+
# Get the coordinate string for a package
57+
Function Get-ClearlyDefinedCoordinates {
58+
[CmdletBinding()]
59+
param(
60+
[validateset('nuget')]
61+
[string]
62+
$PackageType = 'nuget',
63+
[parameter(mandatory = $true)]
64+
[string]
65+
$PackageName,
66+
[parameter(mandatory = $true)]
67+
[string]
68+
$PackageVersion
69+
)
70+
71+
return "$PackageType/$PackageType/-/$PackageName/$PackageVersion"
72+
}
73+
74+
# Cache of ClearlyDefined data
75+
$cdCache = @{}
76+
77+
# Get the ClearlyDefined data for a package
78+
Function Get-ClearlyDefinedData {
79+
[CmdletBinding()]
80+
param(
81+
[Parameter(ValueFromPipelineByPropertyName=$true)]
82+
[Alias('Type')]
83+
[validateset('nuget')]
84+
[string]
85+
$PackageType = 'nuget',
86+
87+
[parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
88+
[Alias('Name')]
89+
[string]
90+
$PackageName,
91+
92+
[parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
93+
[Alias('Revision')]
94+
[string]
95+
$PackageVersion
96+
)
97+
98+
Begin {
99+
$cacheMinutes = 60
100+
$cacheCutoff = (get-date).AddMinutes(-$cacheMinutes)
101+
$coordinateList = @()
102+
}
103+
104+
Process {
105+
$coordinateList += Get-ClearlyDefinedCoordinates @PSBoundParameters
106+
}
107+
108+
end {
109+
$total = $coordinateList.Count
110+
$completed = 0
111+
foreach($coordinates in $coordinateList) {
112+
Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates" -PercentComplete (($completed / $total) * 100)
113+
$containsKey = $cdCache.ContainsKey($coordinates)
114+
if ($containsKey -and $cdCache[$coordinates].cachedTime -gt $cacheCutoff) {
115+
Write-Verbose "Returning cached data for $coordinates"
116+
Write-Output $cdCache[$coordinates]
117+
continue
118+
}
119+
120+
Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" | ForEach-Object {
121+
[bool] $harvested = if ($_.licensed.declared) { $true } else { $false }
122+
Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru | Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru
123+
if ($_.harvested) {
124+
Write-Verbose "Caching data for $coordinates"
125+
$cdCache[$coordinates] = $_
126+
}
127+
}
128+
$completed++
129+
}
130+
}
131+
}
132+
133+
Export-ModuleMember -Function @(
134+
'Start-ClearlyDefinedHarvest'
135+
'Get-ClearlyDefinedData'
136+
'ConvertFrom-ClearlyDefinedCoordinates'
137+
)

0 commit comments

Comments
 (0)