Added tool selection from referenced platform

Author: cmaglie

arduino/cores/packagemanager/package_manager.go

@@ -614,11 +614,9 @@ func (pme *Explorer) GetTool(toolID string) *cores.Tool {
 	}
 }
 
-// FindToolsRequiredForBoard FIXMEDOC
-func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.ToolRelease, error) {
-	pme.log.Infof("Searching tools required for board %s", board)
-
-	platform := board.PlatformRelease
+// FindToolsRequiredForBuild returns the list of ToolReleases needed to build for the specified
+// plaftorm. The buildPlatform may be different depending on the selected board.
+func (pme *Explorer) FindToolsRequiredForBuild(platform, buildPlatform *cores.PlatformRelease) ([]*cores.ToolRelease, error) {
 
 	// maps tool name => all available ToolRelease
 	allToolsAlternatives := map[string][]*cores.ToolRelease{}
@@ -631,14 +629,19 @@ func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.Too
 	// selectBest select the tool with best matching, applying the following rules
 	// in order of priority:
 	// - the tool comes from the requested packager
+	// - the tool comes from the build platform packager
 	// - the tool has the greatest version
 	// - the tool packager comes first in alphabetic order
-	selectBest := func(tools []*cores.ToolRelease, packager string) *cores.ToolRelease {
+	packagerPriority := map[string]int{}
+	packagerPriority[platform.Platform.Package.Name] = 2
+	if buildPlatform != nil {
+		packagerPriority[buildPlatform.Platform.Package.Name] = 1
+	}
+	selectBest := func(tools []*cores.ToolRelease) *cores.ToolRelease {
 		selected := tools[0]
-		namePriority := map[string]int{packager: 1}
 		for _, tool := range tools[1:] {
-			if namePriority[tool.Tool.Package.Name] != namePriority[selected.Tool.Package.Name] {
-				if namePriority[tool.Tool.Package.Name] > namePriority[selected.Tool.Package.Name] {
+			if packagerPriority[tool.Tool.Package.Name] != packagerPriority[selected.Tool.Package.Name] {
+				if packagerPriority[tool.Tool.Package.Name] > packagerPriority[selected.Tool.Package.Name] {
 					selected = tool
 				}
 				continue
@@ -662,7 +665,7 @@ func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.Too
 	// on more than one version of the same tool. For example adafruit:samd has both
 	// [email protected] and [email protected]. To allow the runtime property
 	// {runtime.tools.bossac.path} to be correctly set to the 1.8.0 version we must ensure
-	// taht the returned array is sorted by version.
+	// that the returned array is sorted by version.
 	platform.ToolDependencies.Sort()
 	for _, toolDep := range platform.ToolDependencies {
 		pme.log.WithField("tool", toolDep).Infof("Required tool")
@@ -676,9 +679,9 @@ func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.Too
 
 	// Since a Platform may not specify the required tools (because it's a platform that comes
 	// from a user/hardware dir without a package_index.json) then add all available tools giving
-	// priority to tools coming from the same packager
+	// priority to tools coming from the same packager or referenced packager
 	for _, tools := range allToolsAlternatives {
-		tool := selectBest(tools, platform.Platform.Package.Name)
+		tool := selectBest(tools)
 		requiredTools = append(requiredTools, tool)
 	}
 	return requiredTools, nil

arduino/cores/packagemanager/package_manager_test.go

@@ -287,6 +287,8 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	loadIndex("https://dl.espressif.com/dl/package_esp32_index.json")
 	loadIndex("http://arduino.esp8266.com/stable/package_esp8266com_index.json")
 	loadIndex("https://adafruit.github.io/arduino-board-index/package_adafruit_index.json")
+	loadIndex("https://test.com/package_test_index.json")
+
 	// We ignore the errors returned since they might not be necessarily blocking
 	// but just warnings for the user, like in the case a board is not loaded
 	// because of malformed menus
@@ -310,8 +312,13 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	})
 	require.NotNil(t, esptool0413)
 
+	testPlatform := pme.FindPlatformRelease(&packagemanager.PlatformReference{
+		Package:              "test",
+		PlatformArchitecture: "avr",
+		PlatformVersion:      semver.MustParse("1.1.0")})
+
 	testConflictingToolsInDifferentPackages := func() {
-		tools, err := pme.FindToolsRequiredForBoard(esp32)
+		tools, err := pme.FindToolsRequiredForBuild(esp32.PlatformRelease, nil)
 		require.NoError(t, err)
 		require.Contains(t, tools, esptool231)
 		require.NotContains(t, tools, esptool0413)
@@ -331,10 +338,44 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	testConflictingToolsInDifferentPackages()
 	testConflictingToolsInDifferentPackages()
 
+	{
+		// Test buildPlatform dependencies
+		arduinoBossac180 := pme.FindToolDependency(&cores.ToolDependency{
+			ToolPackager: "arduino",
+			ToolName:     "bossac",
+			ToolVersion:  semver.ParseRelaxed("1.8.0-48-gb176eee"),
+		})
+		require.NotNil(t, arduinoBossac180)
+		testBossac175 := pme.FindToolDependency(&cores.ToolDependency{
+			ToolPackager: "test",
+			ToolName:     "bossac",
+			ToolVersion:  semver.ParseRelaxed("1.7.5"),
+		})
+		require.NotNil(t, testBossac175)
+
+		tools, err := pme.FindToolsRequiredForBuild(esp32.PlatformRelease, nil)
+		require.NoError(t, err)
+		require.Contains(t, tools, esptool231)
+		require.NotContains(t, tools, esptool0413)
+		// When building without testPlatform dependency, arduino:bossac should be selected
+		// since it has the higher version
+		require.NotContains(t, tools, testBossac175)
+		require.Contains(t, tools, arduinoBossac180)
+
+		tools, err = pme.FindToolsRequiredForBuild(esp32.PlatformRelease, testPlatform)
+		require.NoError(t, err)
+		require.Contains(t, tools, esptool231)
+		require.NotContains(t, tools, esptool0413)
+		// When building with testPlatform dependency, test:bossac should be selected
+		// because it has dependency priority
+		require.Contains(t, tools, testBossac175)
+		require.NotContains(t, tools, arduinoBossac180)
+	}
+
 	feather, err := pme.FindBoardWithFQBN("adafruit:samd:adafruit_feather_m0_express")
 	require.NoError(t, err)
 	require.NotNil(t, feather)
-	featherTools, err := pme.FindToolsRequiredForBoard(feather)
+	featherTools, err := pme.FindToolsRequiredForBuild(feather.PlatformRelease, nil)
 	require.NoError(t, err)
 	require.NotNil(t, featherTools)
 

arduino/cores/packagemanager/testdata/data_dir_1/package_test_index.json

@@ -0,0 +1,45 @@
+{
+  "packages": [
+    {
+      "name": "test",
+      "websiteURL": "https://test.com",
+      "email": "[email protected]",
+      "help": {
+        "online": "https://test.com"
+      },
+      "maintainer": "Test",
+      "platforms": [
+        {
+          "architecture": "avr",
+          "boards": [],
+          "name": "Test AVR Boards",
+          "category": "Test",
+          "version": "1.1.0",
+          "archiveFileName": "test-1.1.0.tar.bz2",
+          "checksum": "SHA-256:4e72d4267d9a8d86874edcd021dc661854a5136c0eed947a6fe10366bc51e373",
+          "help": {
+            "online": "https://test.com"
+          },
+          "url": "https://test.com/test-1.1.0.tar.bz2",
+          "toolsDependencies": [],
+          "size": "12345"
+        }
+      ],
+      "tools": [
+        {
+          "name": "bossac",
+          "version": "1.7.5",
+          "systems": [
+            {
+              "host": "i386-apple-darwin11",
+              "checksum": "MD5:603bcce8e59683ac27054b3197a53254",
+              "size": "96372129",
+              "archiveFileName": "bossac.tar.bz2",
+              "url": "https://test.com/bossac.tar.bz2"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

arduino/cores/packagemanager/testdata/data_dir_1/packages/test/hardware/avr/1.1.0/boards.txt

@@ -66,7 +66,7 @@ func getDebugProperties(req *debug.DebugConfigRequest, pme *packagemanager.Explo
 	}
 
 	// Find target board and board properties
-	_, platformRelease, board, boardProperties, referencedPlatformRelease, err := pme.ResolveFQBN(fqbn)
+	_, platformRelease, _, boardProperties, referencedPlatformRelease, err := pme.ResolveFQBN(fqbn)
 	if err != nil {
 		return nil, &arduino.UnknownFQBNError{Cause: err}
 	}
@@ -98,7 +98,7 @@ func getDebugProperties(req *debug.DebugConfigRequest, pme *packagemanager.Explo
 	for _, tool := range pme.GetAllInstalledToolsReleases() {
 		toolProperties.Merge(tool.RuntimeProperties())
 	}
-	if requiredTools, err := pme.FindToolsRequiredForBoard(board); err == nil {
+	if requiredTools, err := pme.FindToolsRequiredForBuild(platformRelease, referencedPlatformRelease); err == nil {
 		for _, requiredTool := range requiredTools {
 			logrus.WithField("tool", requiredTool).Info("Tool required for debug")
 			toolProperties.Merge(requiredTool.RuntimeProperties())

arduino/cores/packagemanager/testdata/data_dir_1/packages/test/tools/bossac/1.7.5/.keep

@@ -295,7 +295,7 @@ func runProgramAction(pme *packagemanager.Explorer,
 	for _, tool := range pme.GetAllInstalledToolsReleases() {
 		uploadProperties.Merge(tool.RuntimeProperties())
 	}
-	if requiredTools, err := pme.FindToolsRequiredForBoard(board); err == nil {
+	if requiredTools, err := pme.FindToolsRequiredForBuild(boardPlatform, buildPlatform); err == nil {
 		for _, requiredTool := range requiredTools {
 			logrus.WithField("tool", requiredTool).Info("Tool required for upload")
 			if requiredTool.IsInstalled() {

commands/debug/debug_info.go

@@ -74,6 +74,23 @@ plus `Name`, `Version`, and an `UpToDate` boolean flag.
 
 `InstallZipLib` method `archivePath` is now a `paths.Path` instead of a `string`.
 
+### golang API change in `github.com/arduino/arduino-cli/rduino/cores/packagemanager.Explorer`
+
+The `packagemanager.Explorer` method `FindToolsRequiredForBoard`:
+
+```go
+func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.ToolRelease, error) { ... }
+```
+
+has been renamed to `FindToolsRequiredForBuild:
+
+```go
+func (pme *Explorer) FindToolsRequiredForBuild(platform, buildPlatform *cores.PlatformRelease) ([]*cores.ToolRelease, error) { ... }
+```
+
+moreover it now requires the `platform` and the `buildPlatform` (a.k.a. the referenced platform core used for the
+compile) instead of the `board`. Usually these two value are obtained from the `Explorer.ResolveFQBN(...)` method.
+
 ## 0.29.0
 
 ### Removed gRPC API: `cc.arduino.cli.commands.v1.UpdateCoreLibrariesIndex`, `Outdated`, and `Upgrade`

commands/upload/upload.go

@@ -278,11 +278,17 @@ are as follows:
 
   1. the tool, version and packager specified via `toolsDependencies` in the `package_index.json`
   1. the highest version of the tool provided by the packager of the current platform
+  1. the highest version of the tool provided by the packager of the referenced platform used for compile (see
+     ["Referencing another core, variant or tool"](platform-specification.md#referencing-another-core-variant-or-tool)
+     for more info)
   1. the highest version of the tool provided by any other packager (in case of tie, the first packager in alphabetical
      order wins)
 
 - The property `{runtime.tools.TOOLNAME-VERSION.path}` points, in order of priority, to:
   1. the tool and version provided by the packager of the current platform
+  1. the tool and version provided by the packager of the referenced platform used for compile (see
+     ["Referencing another core, variant or tool"](platform-specification.md#referencing-another-core-variant-or-tool)
+     for more info)
   1. the tool and version provided by any other packager (in case of tie, the first packager in alphabetical order wins)
 
 ### Example JSON index file

docs/UPGRADING.md

@@ -25,7 +25,7 @@ import (
 type TargetBoardResolver struct{}
 
 func (s *TargetBoardResolver) Run(ctx *types.Context) error {
-	targetPackage, targetPlatform, targetBoard, buildProperties, actualPlatform, err := ctx.PackageManager.ResolveFQBN(ctx.FQBN)
+	targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := ctx.PackageManager.ResolveFQBN(ctx.FQBN)
 	if err != nil {
 		return fmt.Errorf("%s: %w", tr("Error resolving FQBN"), err)
 	}
@@ -39,7 +39,7 @@ func (s *TargetBoardResolver) Run(ctx *types.Context) error {
 
 	if ctx.Verbose {
 		ctx.Info(tr("Using board '%[1]s' from platform in folder: %[2]s", targetBoard.BoardID, targetPlatform.InstallDir))
-		ctx.Info(tr("Using core '%[1]s' from platform in folder: %[2]s", core, actualPlatform.InstallDir))
+		ctx.Info(tr("Using core '%[1]s' from platform in folder: %[2]s", core, buildPlatform.InstallDir))
 	}
 
 	if buildProperties.Get("build.board") == "" {
@@ -50,7 +50,7 @@ func (s *TargetBoardResolver) Run(ctx *types.Context) error {
 			targetBoard.String(), "'build.board'", defaultBuildBoard))
 	}
 
-	requiredTools, err := ctx.PackageManager.FindToolsRequiredForBoard(targetBoard)
+	requiredTools, err := ctx.PackageManager.FindToolsRequiredForBuild(targetPlatform, buildPlatform)
 	if err != nil {
 		return err
 	}
@@ -60,7 +60,7 @@ func (s *TargetBoardResolver) Run(ctx *types.Context) error {
 	ctx.TargetBoardBuildProperties = buildProperties
 	ctx.TargetPlatform = targetPlatform
 	ctx.TargetPackage = targetPackage
-	ctx.ActualPlatform = actualPlatform
+	ctx.ActualPlatform = buildPlatform
 	ctx.RequiredTools = requiredTools
 	return nil
 }