Add support for file:// schema to set local additional URLs

cli/cli.go

@@ -104,7 +104,6 @@ func createCliCommandTree(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", "The output format, can be {text|json}.")
 	cmd.PersistentFlags().StringVar(&configFile, "config-file", "", "The custom config file (if not specified the default will be used).")
 	cmd.PersistentFlags().StringSlice("additional-urls", []string{}, "Comma-separated list of additional URLs for the Boards Manager.")
-	cmd.PersistentFlags().StringSlice("additional-paths", []string{}, "Comma-separated list of additional file paths for the Boards Manager.")
 	configuration.BindFlags(cmd, configuration.Settings)
 }
 

cli/config/validate.go

@@ -21,19 +21,18 @@ import (
 )
 
 var validMap = map[string]reflect.Kind{
-	"board_manager.additional_urls":  reflect.Slice,
-	"board_manager.additional_paths": reflect.Slice,
-	"daemon.port":                    reflect.String,
-	"directories.data":               reflect.String,
-	"directories.downloads":          reflect.String,
-	"directories.user":               reflect.String,
-	"library.enable_unsafe_install":  reflect.Bool,
-	"logging.file":                   reflect.String,
-	"logging.format":                 reflect.String,
-	"logging.level":                  reflect.String,
-	"sketch.always_export_binaries":  reflect.Bool,
-	"telemetry.addr":                 reflect.String,
-	"telemetry.enabled":              reflect.Bool,
+	"board_manager.additional_urls": reflect.Slice,
+	"daemon.port":                   reflect.String,
+	"directories.data":              reflect.String,
+	"directories.downloads":         reflect.String,
+	"directories.user":              reflect.String,
+	"library.enable_unsafe_install": reflect.Bool,
+	"logging.file":                  reflect.String,
+	"logging.format":                reflect.String,
+	"logging.level":                 reflect.String,
+	"sketch.always_export_binaries": reflect.Bool,
+	"telemetry.addr":                reflect.String,
+	"telemetry.enabled":             reflect.Bool,
 }
 
 func typeOf(key string) (reflect.Kind, error) {

commands/daemon/testdata/arduino-cli.yml

@@ -2,7 +2,6 @@ board_manager:
   additional_urls:
     - http://foobar.com
     - http://example.com
-  additional_paths: []
 
 daemon:
   port: "50051"

commands/instances.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"runtime"
 
 	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/cores"
@@ -202,24 +203,6 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo
 
 	indexpath := paths.New(configuration.Settings.GetString("directories.Data"))
 
-	for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
-		logrus.Info("JSON PATH: ", x)
-
-		pathJSON, _ := paths.New(x).Abs()
-
-		if _, err := packageindex.LoadIndexNoSign(pathJSON); err != nil {
-			return nil, fmt.Errorf("invalid package index in %s: %s", pathJSON, err)
-		}
-
-		fi, _ := os.Stat(x)
-		downloadCB(&rpc.DownloadProgress{
-			File:      "Updating index: " + pathJSON.Base(),
-			TotalSize: fi.Size(),
-		})
-		downloadCB(&rpc.DownloadProgress{Completed: true})
-
-	}
-
 	urls := []string{globals.DefaultIndexURL}
 	urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
 	for _, u := range urls {
@@ -232,6 +215,31 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo
 
 		logrus.WithField("url", URL).Print("Updating index")
 
+		if URL.Scheme == "file" {
+			path := paths.New(URL.Path)
+			if runtime.GOOS == "windows" {
+				// Parsed local file URLs on Windows are returned with a leading /
+				// so we remove it
+				path = paths.New(URL.Path[1:])
+			}
+			pathJSON, err := path.Abs()
+			if err != nil {
+				return nil, fmt.Errorf("can't get absolute path of %v: %w", path, err)
+			}
+
+			if _, err := packageindex.LoadIndexNoSign(pathJSON); err != nil {
+				return nil, fmt.Errorf("invalid package index in %s: %s", pathJSON, err)
+			}
+
+			fi, _ := os.Stat(path.String())
+			downloadCB(&rpc.DownloadProgress{
+				File:      "Updating index: " + pathJSON.Base(),
+				TotalSize: fi.Size(),
+			})
+			downloadCB(&rpc.DownloadProgress{Completed: true})
+			continue
+		}
+
 		var tmp *paths.Path
 		if tmpFile, err := ioutil.TempFile("", ""); err != nil {
 			return nil, fmt.Errorf("creating temp file for index download: %s", err)
@@ -665,16 +673,26 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult
 				continue
 			}
 
-			if err := res.Pm.LoadPackageIndex(URL); err != nil {
-				res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
-			}
-		}
+			if URL.Scheme == "file" {
+				path := paths.New(URL.Path)
+				if runtime.GOOS == "windows" {
+					// Parsed local file URLs on Windows are returned with a leading /
+					// so we remove it
+					path = paths.New(URL.Path[1:])
+				}
+				pathJSON, err := path.Abs()
+				if err != nil {
+					return nil, fmt.Errorf("can't get absolute path of %v: %w", path, err)
+				}
 
-		for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
-			pathJSON, _ := paths.New(x).Abs()
+				_, err = res.Pm.LoadPackageIndexFromFile(pathJSON)
+				if err != nil {
+					res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
+				}
+				continue
+			}
 
-			_, err := res.Pm.LoadPackageIndexFromFile(pathJSON)
-			if err != nil {
+			if err := res.Pm.LoadPackageIndex(URL); err != nil {
 				res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
 			}
 		}

configuration/configuration.go

@@ -76,7 +76,6 @@ func BindFlags(cmd *cobra.Command, settings *viper.Viper) {
 	settings.BindPFlag("logging.file", cmd.Flag("log-file"))
 	settings.BindPFlag("logging.format", cmd.Flag("log-format"))
 	settings.BindPFlag("board_manager.additional_urls", cmd.Flag("additional-urls"))
-	settings.BindPFlag("board_manager.additional_paths", cmd.Flag("additional-paths"))
 }
 
 // getDefaultArduinoDataDir returns the full path to the default arduino folder

configuration/configuration_test.go

@@ -89,7 +89,6 @@ func TestInit(t *testing.T) {
 	require.Equal(t, "text", settings.GetString("logging.format"))
 
 	require.Empty(t, settings.GetStringSlice("board_manager.additional_urls"))
-	require.Empty(t, settings.GetStringSlice("board_manager.additional_paths"))
 
 	require.NotEmpty(t, settings.GetString("directories.Data"))
 	require.NotEmpty(t, settings.GetString("directories.Downloads"))

configuration/defaults.go

@@ -33,7 +33,6 @@ func SetDefaults(settings *viper.Viper) {
 
 	// Boards Manager
 	settings.SetDefault("board_manager.additional_urls", []string{})
-	settings.SetDefault("board_manager.additional_paths", []string{})
 
 	// arduino directories
 	settings.SetDefault("directories.Data", getDefaultArduinoDataDir())

docs/configuration.md

@@ -2,8 +2,6 @@
 
 - `board_manager`
   - `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms.
-  - `additional_paths` - the absolute file paths to any additional Boards Manager package index files needed for your
-    boards platforms.
 - `daemon` - options related to running Arduino CLI as a [gRPC] server.
   - `port` - TCP port used for gRPC client connections.
 - `directories` - directories used by Arduino CLI.
@@ -68,12 +66,6 @@ Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITI
 $ export ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://downloads.arduino.cc/packages/package_staging_index.json
 ```
 
-Setting an additional Boards Manager file using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable:
-
-```sh
-$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=/path/to/your/package_staging_index.json
-```
-
 ### Configuration file
 
 [`arduino-cli config init`][arduino-cli config init] creates or updates a configuration file with the current
@@ -136,21 +128,6 @@ Doing the same using a TOML format file:
 additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index.json" ]
 ```
 
-Setting an additional Boards Manager File using a YAML format configuration file:
-
-```yaml
-board_manager:
-  additional_paths:
-    - /path/to/your/package_staging_index.json
-```
-
-Doing the same using a TOML format file:
-
-```toml
-[board_manager]
-additional_paths = [ "/path/to/your/package_staging_index.json" ]
-```
-
 [grpc]: https://grpc.io
 [sketchbook directory]: sketch-specification.md#sketchbook
 [arduino cli lib install]: commands/arduino-cli_lib_install.md

docs/getting-started.md

@@ -183,12 +183,12 @@ For example, to add the NRF52832 core, edit the configuration file and change th
 
 ```yaml
 board_manager:
-  additional_paths:
-    - /absolute/path/to/your/package_nrf52832_index.json
+  additional_urls:
+    - https://arduino.esp8266.com/stable/package_esp8266com_index.json
+    - file:///absolute/path/to/your/package_nrf52832_index.json
 ```
 
-From now on, commands supporting custom cores will automatically use the additional URL and additional paths from the
-configuration file:
+From now on, commands supporting custom cores will automatically use the additional URL from the configuration file:
 
 ```sh
 $ arduino-cli core update-index
@@ -214,14 +214,13 @@ ID              Version Name
 esp8266:esp8266 2.5.2   esp8266
 ```
 
-The same applies to the additional package index file provided by file paths. Use the `--additional-paths` option, that
-has to be specified every time and for every command that operates on a 3rd party platform core, for example:
+The same applies to the additional package index file provided by file paths:
 
 ```sh
-$ arduino-cli  core update-index --additional-paths /absolute/path/to/your/package_esp8266com_index.json
+$ arduino-cli  core update-index --additional-urls file:///absolute/path/to/your/package_esp8266com_index.json
 Updating index: package_esp8266com_index.json downloaded
 
-$ arduino-cli core search esp8266 --additional-paths /absolute/path/to/your/package_esp8266com_index.json
+$ arduino-cli core search esp8266 --additional-urls file:///absolute/path/to/your/package_esp8266com_index.json
 ID              Version Name
 esp8266:esp8266 2.5.2   esp8266
 ```

test/test_core.py

@@ -353,3 +353,13 @@ def ordered(obj):
             return obj
 
     assert ordered(installed_json) == ordered(expected_installed_json)
+
+
+def test_core_update_with_local_url(run_command):
+    test_index = str(Path(__file__).parent / "testdata" / "test_index.json")
+    if platform.system() == "Windows":
+        test_index = f"/{test_index}".replace("\\", "/")
+
+    res = run_command(f'core update-index --additional-urls="file://{test_index}"')
+    assert res.ok
+    assert "Updating index: test_index.json downloaded" in res.stdout