From 1c43f2f2ca346832d2161648f59d2a003c75b989 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Sun, 7 Nov 2021 23:16:52 -0500 Subject: [PATCH 1/8] Parse the constituent files --- .../Registry/Scripts/FindDroppedPackages.purs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 ci/src/Registry/Scripts/FindDroppedPackages.purs diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs new file mode 100644 index 000000000..62b533c7c --- /dev/null +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -0,0 +1,55 @@ +module Registry.Scripts.FindDroppedPackages where + +import Registry.Prelude + +import Effect.Console (logShow) +import Data.Argonaut as Json +import Effect.Aff as Aff +import Registry.Scripts.LegacyImport.Error (PackageFailures(..), RawPackageName(..), RawVersion(..)) + +main :: Effect Unit +main = Aff.launchAff_ do + bowerExclusionsFile <- readJsonFile "bower-exclusions.json" + case bowerExclusionsFile of + Left err -> do + let decodeError = "Decoding bower-exlusions.json failed with error:\n\n" <> Json.printJsonDecodeError err + throwError $ Aff.error decodeError + Right bowerExclusions -> do + packageSetPackagesFile <- readJsonFile "../../package-sets/packages.json" + case packageSetPackagesFile of + Left err -> do + let decodeError = "Decoding packages.json failed with error:\n\n" <> Json.printJsonDecodeError err + throwError $ Aff.error decodeError + Right packages -> + let + packagesThatWillBeDropped = findPackagesThatWillBeDropped packages bowerExclusions + in + pure unit + +-- pure unit + +newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) + +instance Json.DecodeJson PackageSet where + decodeJson json = do + packagesObject :: Object PackageSetPackage <- Json.decodeJson json + pure + $ PackageSet + $ objectToMap (Just <<< RawPackageName) packagesObject + +-- type PackageSetJson = Map String { version :: String } + +-- type PackageSetJson = Map RawPackageName PackageSetPackage + +type PackageSetPackage = + { version :: RawVersion + } + +findPackagesThatWillBeDropped :: PackageSet -> PackageFailures -> Array String +findPackagesThatWillBeDropped packageSet bowerExclusions = [] +-- bowerExclusions +-- # un PackageFailures +-- # Map.values +-- # List.filter \package + +-- willBeDropped :: PackageSetJson -> Either RawPackageName \ No newline at end of file From 54d40d6fe921e0fe622b07ccf70e37a781fa7ea0 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Sun, 7 Nov 2021 23:40:20 -0500 Subject: [PATCH 2/8] Log the packages that will be dropped --- .../Registry/Scripts/FindDroppedPackages.purs | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index 62b533c7c..b16fd6c87 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -4,15 +4,18 @@ import Registry.Prelude import Effect.Console (logShow) import Data.Argonaut as Json +import Data.Array as Array +import Data.List as List +import Data.Map as Map import Effect.Aff as Aff -import Registry.Scripts.LegacyImport.Error (PackageFailures(..), RawPackageName(..), RawVersion(..)) +import Registry.Scripts.LegacyImport.Error (ImportError, PackageFailures(..), RawPackageName(..), RawVersion(..)) main :: Effect Unit main = Aff.launchAff_ do bowerExclusionsFile <- readJsonFile "bower-exclusions.json" case bowerExclusionsFile of Left err -> do - let decodeError = "Decoding bower-exlusions.json failed with error:\n\n" <> Json.printJsonDecodeError err + let decodeError = "Decoding bower-exclusions.json failed with error:\n\n" <> Json.printJsonDecodeError err throwError $ Aff.error decodeError Right bowerExclusions -> do packageSetPackagesFile <- readJsonFile "../../package-sets/packages.json" @@ -22,14 +25,14 @@ main = Aff.launchAff_ do throwError $ Aff.error decodeError Right packages -> let - packagesThatWillBeDropped = findPackagesThatWillBeDropped packages bowerExclusions + packagesThatWillBeDropped = findPackagesThatWillBeDropped packages $ dropImportErrorKeys bowerExclusions in - pure unit - --- pure unit + liftEffect $ logShow $ map (un RawPackageName) packagesThatWillBeDropped newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) +derive instance Newtype PackageSet _ + instance Json.DecodeJson PackageSet where decodeJson json = do packagesObject :: Object PackageSetPackage <- Json.decodeJson json @@ -37,19 +40,30 @@ instance Json.DecodeJson PackageSet where $ PackageSet $ objectToMap (Just <<< RawPackageName) packagesObject --- type PackageSetJson = Map String { version :: String } - --- type PackageSetJson = Map RawPackageName PackageSetPackage - type PackageSetPackage = { version :: RawVersion } -findPackagesThatWillBeDropped :: PackageSet -> PackageFailures -> Array String -findPackagesThatWillBeDropped packageSet bowerExclusions = [] --- bowerExclusions --- # un PackageFailures --- # Map.values --- # List.filter \package +type ExcludedPackages = Map RawPackageName (Either ImportError (Map RawVersion ImportError)) + +dropImportErrorKeys :: PackageFailures -> ExcludedPackages +dropImportErrorKeys = + un PackageFailures + >>> Map.values + >>> List.foldl Map.union Map.empty --- willBeDropped :: PackageSetJson -> Either RawPackageName \ No newline at end of file +findPackagesThatWillBeDropped :: PackageSet -> ExcludedPackages -> Array RawPackageName +findPackagesThatWillBeDropped packageSet bowerExclusions = + packageSet + # un PackageSet + # Map.toUnfoldable + # Array.mapMaybe \(Tuple name { version }) -> willBeDropped name version + where + willBeDropped name version = + case Map.lookup name bowerExclusions of + Just (Left _) -> Just name + Just (Right excludedVersions) -> + case Map.lookup version excludedVersions of + Just _ -> Just name + Nothing -> Nothing + Nothing -> Nothing From 621f045b07b93d25f074eaefbb2c45ec690fbc9e Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 00:00:39 -0500 Subject: [PATCH 3/8] Improve output --- .../Registry/Scripts/FindDroppedPackages.purs | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index b16fd6c87..0da8daf8a 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -5,10 +5,35 @@ import Registry.Prelude import Effect.Console (logShow) import Data.Argonaut as Json import Data.Array as Array +import Data.Interpolate (i) import Data.List as List import Data.Map as Map import Effect.Aff as Aff -import Registry.Scripts.LegacyImport.Error (ImportError, PackageFailures(..), RawPackageName(..), RawVersion(..)) +import Registry.Scripts.LegacyImport.Error (ImportError, ImportErrorKey(..), PackageFailures(..), RawPackageName(..), RawVersion(..)) +import Registry.Scripts.LegacyImport.Error as Error + +newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) + +derive instance Newtype PackageSet _ + +instance Json.DecodeJson PackageSet where + decodeJson json = do + packagesObject :: Object PackageSetPackage <- Json.decodeJson json + pure + $ PackageSet + $ objectToMap (Just <<< RawPackageName) packagesObject + +type PackageSetPackage = + { version :: RawVersion + } + +type DroppedPackage = + { name :: RawPackageName + , version :: RawVersion + , reason :: ImportError + } + +type ExcludedPackages = Map RawPackageName (Either ImportError (Map RawVersion ImportError)) main :: Effect Unit main = Aff.launchAff_ do @@ -27,24 +52,17 @@ main = Aff.launchAff_ do let packagesThatWillBeDropped = findPackagesThatWillBeDropped packages $ dropImportErrorKeys bowerExclusions in - liftEffect $ logShow $ map (un RawPackageName) packagesThatWillBeDropped - -newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) - -derive instance Newtype PackageSet _ - -instance Json.DecodeJson PackageSet where - decodeJson json = do - packagesObject :: Object PackageSetPackage <- Json.decodeJson json - pure - $ PackageSet - $ objectToMap (Just <<< RawPackageName) packagesObject + liftEffect $ packagesThatWillBeDropped + # traverse printMessage -type PackageSetPackage = - { version :: RawVersion - } - -type ExcludedPackages = Map RawPackageName (Either ImportError (Map RawVersion ImportError)) +printMessage :: DroppedPackage -> Effect Unit +printMessage droppedPackage = log message + where + name = un RawPackageName droppedPackage.name + version = un RawVersion droppedPackage.version + reason = un ImportErrorKey $ Error.printImportErrorKey droppedPackage.reason + packageIdentifier = name <> " " <> version + message = i packageIdentifier " will be dropped from the package set due to: " <> reason dropImportErrorKeys :: PackageFailures -> ExcludedPackages dropImportErrorKeys = @@ -52,7 +70,7 @@ dropImportErrorKeys = >>> Map.values >>> List.foldl Map.union Map.empty -findPackagesThatWillBeDropped :: PackageSet -> ExcludedPackages -> Array RawPackageName +findPackagesThatWillBeDropped :: PackageSet -> ExcludedPackages -> Array DroppedPackage findPackagesThatWillBeDropped packageSet bowerExclusions = packageSet # un PackageSet @@ -61,9 +79,9 @@ findPackagesThatWillBeDropped packageSet bowerExclusions = where willBeDropped name version = case Map.lookup name bowerExclusions of - Just (Left _) -> Just name + Just (Left reason) -> Just { name, version, reason } Just (Right excludedVersions) -> case Map.lookup version excludedVersions of - Just _ -> Just name + Just reason -> Just { name, version, reason } Nothing -> Nothing Nothing -> Nothing From 18118d19eb07953da290f668e261ada07f879d63 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 21:05:44 -0500 Subject: [PATCH 4/8] Improve error output --- ci/spago.dhall | 1 + .../Registry/Scripts/FindDroppedPackages.purs | 50 ++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ci/spago.dhall b/ci/spago.dhall index 68806b194..bb23674ba 100644 --- a/ci/spago.dhall +++ b/ci/spago.dhall @@ -47,6 +47,7 @@ , "spec" , "string-parsers" , "strings" + , "stringutils" , "sunde" , "transformers" , "tuples" diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index 0da8daf8a..974f63f0c 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -5,12 +5,20 @@ import Registry.Prelude import Effect.Console (logShow) import Data.Argonaut as Json import Data.Array as Array +import Data.Foldable (intercalate) import Data.Interpolate (i) import Data.List as List import Data.Map as Map +import Data.String.Utils (lines) import Effect.Aff as Aff -import Registry.Scripts.LegacyImport.Error (ImportError, ImportErrorKey(..), PackageFailures(..), RawPackageName(..), RawVersion(..)) -import Registry.Scripts.LegacyImport.Error as Error +import Registry.Scripts.LegacyImport.Error + ( ImportError(..) + , ImportErrorKey(..) + , ManifestError(..) + , PackageFailures(..) + , RawPackageName(..) + , RawVersion(..) + ) newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) @@ -53,16 +61,44 @@ main = Aff.launchAff_ do packagesThatWillBeDropped = findPackagesThatWillBeDropped packages $ dropImportErrorKeys bowerExclusions in liftEffect $ packagesThatWillBeDropped - # traverse printMessage + # map printMessage + # intercalate "\n\n" + # log -printMessage :: DroppedPackage -> Effect Unit -printMessage droppedPackage = log message +printMessage :: DroppedPackage -> String +printMessage droppedPackage = message where name = un RawPackageName droppedPackage.name version = un RawVersion droppedPackage.version - reason = un ImportErrorKey $ Error.printImportErrorKey droppedPackage.reason + reason = printDroppedReason droppedPackage.reason packageIdentifier = name <> " " <> version - message = i packageIdentifier " will be dropped from the package set due to: " <> reason + message = i packageIdentifier " will be dropped from the package set due to:\n\t" <> reason + +printDroppedReason :: ImportError -> String +printDroppedReason = case _ of + InvalidGitHubRepo _ -> "invalid GitHub repo" + ResourceError _ -> "resource error" + MalformedPackageName _ -> "malformed package name" + NoDependencyFiles -> "no dependency files" + NonRegistryDependencies _ -> "non-registry dependencies" + NoManifests -> "no manifests" + ManifestError manifestErrors -> + manifestErrors + # map printManifestError + # intercalate ", " + +printManifestError :: ManifestError -> String +printManifestError = case _ of + MissingName -> "missing name" + MissingLicense -> "missing license" + BadLicense licenses -> + "bad license:\n" <> + ( intercalate "\n" $ map ("\t\t" <> _) + $ Array.concatMap lines licenses + ) + BadVersion version -> "bad version (" <> version <> ")" + InvalidDependencyNames _ -> "invalid dependency names" + BadDependencyVersions _ -> "bad dependency versions" dropImportErrorKeys :: PackageFailures -> ExcludedPackages dropImportErrorKeys = From 8f4db2ef2b045a6e9d1e03c4c9e42f2c794741af Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 21:17:34 -0500 Subject: [PATCH 5/8] Clean up output --- .../Registry/Scripts/FindDroppedPackages.purs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index 974f63f0c..6e38a3b3c 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -2,18 +2,18 @@ module Registry.Scripts.FindDroppedPackages where import Registry.Prelude -import Effect.Console (logShow) import Data.Argonaut as Json import Data.Array as Array import Data.Foldable (intercalate) import Data.Interpolate (i) import Data.List as List import Data.Map as Map +import Data.Array.NonEmpty as NEA import Data.String.Utils (lines) import Effect.Aff as Aff +import Registry.PackageName as PackageName import Registry.Scripts.LegacyImport.Error ( ImportError(..) - , ImportErrorKey(..) , ManifestError(..) , PackageFailures(..) , RawPackageName(..) @@ -96,9 +96,20 @@ printManifestError = case _ of ( intercalate "\n" $ map ("\t\t" <> _) $ Array.concatMap lines licenses ) - BadVersion version -> "bad version (" <> version <> ")" - InvalidDependencyNames _ -> "invalid dependency names" - BadDependencyVersions _ -> "bad dependency versions" + BadVersion version -> "bad version: " <> version + InvalidDependencyNames dependencyNames -> + "invalid dependency names:\n" <> intercalate "\n\t\t" dependencyNames + BadDependencyVersions badVersions -> + "bad dependency versions:\n" <> + ( intercalate "\n" $ map ("\t\t" <> _) + $ Array.concatMap lines + $ map printBadDependencyVersion + $ NEA.toArray badVersions + ) + + where + printBadDependencyVersion { dependency, failedBounds } = + i "Dependency: " (PackageName.print dependency) "\nFailed bounds: " failedBounds dropImportErrorKeys :: PackageFailures -> ExcludedPackages dropImportErrorKeys = From 0c82c339b0e45d365cd7c03f17c8255a3406633c Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 21:22:04 -0500 Subject: [PATCH 6/8] Add doc comments --- ci/src/Registry/Scripts/FindDroppedPackages.purs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index 6e38a3b3c..950588bb3 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -20,6 +20,7 @@ import Registry.Scripts.LegacyImport.Error , RawVersion(..) ) +-- | A PureScript package set. newtype PackageSet = PackageSet (Map RawPackageName PackageSetPackage) derive instance Newtype PackageSet _ @@ -31,10 +32,12 @@ instance Json.DecodeJson PackageSet where $ PackageSet $ objectToMap (Just <<< RawPackageName) packagesObject +-- | A package in the package set. type PackageSetPackage = { version :: RawVersion } +-- | A package that will be dropped from the package set. type DroppedPackage = { name :: RawPackageName , version :: RawVersion @@ -111,12 +114,16 @@ printManifestError = case _ of printBadDependencyVersion { dependency, failedBounds } = i "Dependency: " (PackageName.print dependency) "\nFailed bounds: " failedBounds +-- | Drops the import keys from the `PackageFailures` collection, as we don't +-- | need the groupings. dropImportErrorKeys :: PackageFailures -> ExcludedPackages dropImportErrorKeys = un PackageFailures >>> Map.values >>> List.foldl Map.union Map.empty +-- | Returns an array of all packages that will be dropped from the package set +-- | based on the packages currently excluded from the registry. findPackagesThatWillBeDropped :: PackageSet -> ExcludedPackages -> Array DroppedPackage findPackagesThatWillBeDropped packageSet bowerExclusions = packageSet From 8e2a7b612617be3b6f2838b0c0c23b185d63d28e Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 21:32:57 -0500 Subject: [PATCH 7/8] Update path to package set file --- ci/.gitignore | 3 +++ ci/src/Registry/Scripts/FindDroppedPackages.purs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/.gitignore b/ci/.gitignore index e79001e67..e47f14db5 100644 --- a/ci/.gitignore +++ b/ci/.gitignore @@ -18,3 +18,6 @@ generated-docs/ # Files generated as part of the legacy import process. bower-exclusions.json registry-index + +# A copy of the package set's `packages.json`, for the legacy import process. +package-set-packages.json diff --git a/ci/src/Registry/Scripts/FindDroppedPackages.purs b/ci/src/Registry/Scripts/FindDroppedPackages.purs index 950588bb3..1b0a2bbd2 100644 --- a/ci/src/Registry/Scripts/FindDroppedPackages.purs +++ b/ci/src/Registry/Scripts/FindDroppedPackages.purs @@ -54,7 +54,7 @@ main = Aff.launchAff_ do let decodeError = "Decoding bower-exclusions.json failed with error:\n\n" <> Json.printJsonDecodeError err throwError $ Aff.error decodeError Right bowerExclusions -> do - packageSetPackagesFile <- readJsonFile "../../package-sets/packages.json" + packageSetPackagesFile <- readJsonFile "package-set-packages.json" case packageSetPackagesFile of Left err -> do let decodeError = "Decoding packages.json failed with error:\n\n" <> Json.printJsonDecodeError err From d47cec52972fb0ee5277b9259a8e69cab1b886f8 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 8 Nov 2021 21:33:03 -0500 Subject: [PATCH 8/8] Add instructions --- ci/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/README.md b/ci/README.md index 8ab356aac..24de48799 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,3 +21,21 @@ You can execute the legacy registry import script with the following command: ```console $ spago run -m Registry.Scripts.LegacyImport ``` + +## Find Dropped Packages + +This script reports packages that will be dropped from the package set + +### Setup + +Before running this script you will need to have a `bower-exclusions.json` file (from running the [Legacy Import](#legacy-import)) and a `package-set-packages.json` file: + +```console +$ wget https://raw.githubusercontent.com/purescript/package-sets/master/packages.json -O package-set-packages.json +``` + +With those files present, run the script with the following command: + +```console +$ spago run -m Registry.Scripts.FindDroppedPackages +```