Support for git-downloaded libraries in profiles.

commands/instances.go

@@ -388,6 +388,40 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 				continue
 			}
 
+			if libraryRef.GitURL != nil {
+				uid := libraryRef.InternalUniqueIdentifier()
+				libRoot := s.settings.ProfilesCacheDir().Join(uid)
+				libDir := libRoot.Join(libraryRef.Library)
+
+				if !libDir.IsDir() {
+					// Clone repo and install
+					tmpDir, err := librariesmanager.CloneLibraryGitRepository(ctx, libraryRef.GitURL.String())
+					if err != nil {
+						taskCallback(&rpc.TaskProgress{Name: i18n.Tr("Error downloading library %s", libraryRef)})
+						e := &cmderrors.FailedLibraryInstallError{Cause: err}
+						responseError(e.GRPCStatus())
+						continue
+					}
+
+					// Install library into profile cache
+					copyErr := tmpDir.CopyDirTo(libDir)
+					_ = tmpDir.RemoveAll()
+					if copyErr != nil {
+						taskCallback(&rpc.TaskProgress{Name: i18n.Tr("Error installing library %s", libraryRef)})
+						e := &cmderrors.FailedLibraryInstallError{Cause: fmt.Errorf("copying library to profile cache: %w", err)}
+						responseError(e.GRPCStatus())
+						continue
+					}
+				}
+
+				lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
+					Path:            libDir,
+					Location:        libraries.Profile,
+					IsSingleLibrary: true,
+				})
+				continue
+			}
+
 			uid := libraryRef.InternalUniqueIdentifier()
 			libRoot := s.settings.ProfilesCacheDir().Join(uid)
 			libDir := libRoot.Join(libraryRef.Library)

commands/service_library_install.go

@@ -267,9 +267,8 @@ func (s *arduinoCoreServerImpl) GitLibraryInstall(req *rpc.GitLibraryInstallRequ
 	lmi, release := lm.NewInstaller()
 	defer release()
 
-	// TODO: pass context
-	// ctx := stream.Context()
-	if err := lmi.InstallGitLib(req.GetUrl(), req.GetOverwrite()); err != nil {
+	ctx := stream.Context()
+	if err := lmi.InstallGitLib(ctx, req.GetUrl(), req.GetOverwrite()); err != nil {
 		return &cmderrors.FailedLibraryInstallError{Cause: err}
 	}
 	taskCB(&rpc.TaskProgress{Message: i18n.Tr("Library installed"), Completed: true})

internal/arduino/libraries/librariesmanager/install.go

@@ -200,54 +200,64 @@ func (lmi *Installer) InstallZipLib(ctx context.Context, archivePath *paths.Path
 }
 
 // InstallGitLib installs a library hosted on a git repository on the specified path.
-func (lmi *Installer) InstallGitLib(argURL string, overwrite bool) error {
-	libraryName, gitURL, ref, err := parseGitArgURL(argURL)
+func (lmi *Installer) InstallGitLib(ctx context.Context, argURL string, overwrite bool) error {
+	tmpInstallPath, err := CloneLibraryGitRepository(ctx, argURL)
 	if err != nil {
 		return err
 	}
+	defer tmpInstallPath.RemoveAll()
+
+	// Install extracted library in the destination directory
+	if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil {
+		return errors.New(i18n.Tr("moving extracted archive to destination dir: %s", err))
+	}
+
+	return nil
+}
+
+// CloneLibraryGitRepository clones a git repository containing a library
+// into a temporary directory and returns the path to the cloned library.
+func CloneLibraryGitRepository(ctx context.Context, argURL string) (*paths.Path, error) {
+	libraryName, gitURL, ref, err := parseGitArgURL(argURL)
+	if err != nil {
+		return nil, err
+	}
 
 	// Clone library in a temporary directory
 	tmp, err := paths.MkTempDir("", "")
 	if err != nil {
-		return err
+		return nil, err
 	}
-	defer tmp.RemoveAll()
 	tmpInstallPath := tmp.Join(libraryName)
 
-	if _, err := git.PlainClone(tmpInstallPath.String(), false, &git.CloneOptions{
+	if _, err := git.PlainCloneContext(ctx, tmpInstallPath.String(), false, &git.CloneOptions{
 		URL:           gitURL,
 		ReferenceName: plumbing.ReferenceName(ref),
 	}); err != nil {
 		if err.Error() != "reference not found" {
-			return err
+			return nil, err
 		}
 
 		// We did not find the requested reference, let's do a PlainClone and use
 		// "ResolveRevision" to find and checkout the requested revision
-		if repo, err := git.PlainClone(tmpInstallPath.String(), false, &git.CloneOptions{
+		if repo, err := git.PlainCloneContext(ctx, tmpInstallPath.String(), false, &git.CloneOptions{
 			URL: gitURL,
 		}); err != nil {
-			return err
+			return nil, err
 		} else if h, err := repo.ResolveRevision(plumbing.Revision(ref)); err != nil {
-			return err
+			return nil, err
 		} else if w, err := repo.Worktree(); err != nil {
-			return err
+			return nil, err
 		} else if err := w.Checkout(&git.CheckoutOptions{
 			Force: true, // workaround for: https://github.com/go-git/go-git/issues/1411
 			Hash:  plumbing.NewHash(h.String())}); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	// We don't want the installed library to be a git repository thus we delete this folder
 	tmpInstallPath.Join(".git").RemoveAll()
-
-	// Install extracted library in the destination directory
-	if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil {
-		return errors.New(i18n.Tr("moving extracted archive to destination dir: %s", err))
-	}
-
-	return nil
+	return tmpInstallPath, nil
 }
 
 // parseGitArgURL tries to recover a library name from a git URL.

internal/arduino/sketch/profiles.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -325,23 +326,39 @@ func (p *ProfilePlatformReference) UnmarshalYAML(unmarshal func(interface{}) err
 // ProfileLibraryReference is a reference to a library
 type ProfileLibraryReference struct {
 	Library    string
-	InstallDir *paths.Path
 	Version    *semver.Version
+	InstallDir *paths.Path
+	GitURL     *url.URL
 }
 
 // UnmarshalYAML decodes a ProfileLibraryReference from YAML source.
 func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var dataMap map[string]any
 	if err := unmarshal(&dataMap); err == nil {
-		if installDir, ok := dataMap["dir"]; !ok {
-			return errors.New(i18n.Tr("invalid library reference: %s", dataMap))
-		} else if installDir, ok := installDir.(string); !ok {
-			return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference: %s"), dataMap)
-		} else {
-			l.InstallDir = paths.New(installDir)
-			l.Library = l.InstallDir.Base()
-			return nil
+		if installDir, ok := dataMap["dir"]; ok {
+			if installDir, ok := installDir.(string); !ok {
+				return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference: %s"), dataMap)
+			} else {
+				l.InstallDir = paths.New(installDir)
+				l.Library = l.InstallDir.Base()
+				return nil
+			}
+		}
+		if gitUrl, ok := dataMap["git"]; ok {
+			if gitUrlStr, ok := gitUrl.(string); !ok {
+				return fmt.Errorf("%s: %s", i18n.Tr("invalid git library reference: %s"), dataMap)
+			} else if parsedUrl, err := url.Parse(gitUrlStr); err != nil {
+				return fmt.Errorf("%s: %w", i18n.Tr("invalid git library URL:"), err)
+			} else {
+				l.GitURL = parsedUrl
+				if l.Library = filepath.Base(parsedUrl.Path); l.Library == "" {
+					l.Library = "lib"
+				}
+				l.Library = strings.TrimSuffix(l.Library, ".git")
+				return nil
+			}
 		}
+		return errors.New(i18n.Tr("invalid library reference: %s", dataMap))
 	}
 
 	var data string
@@ -364,12 +381,18 @@ func (l *ProfileLibraryReference) AsYaml() string {
 	if l.InstallDir != nil {
 		return fmt.Sprintf("      - dir: %s\n", l.InstallDir)
 	}
+	if l.GitURL != nil {
+		return fmt.Sprintf("      - git: %s\n", l.GitURL)
+	}
 	return fmt.Sprintf("      - %s (%s)\n", l.Library, l.Version)
 }
 
 func (l *ProfileLibraryReference) String() string {
 	if l.InstallDir != nil {
-		return fmt.Sprintf("%s@dir:%s", l.Library, l.InstallDir)
+		return "@dir:" + l.InstallDir.String()
+	}
+	if l.GitURL != nil {
+		return "@git:" + l.GitURL.String()
 	}
 	return fmt.Sprintf("%s@%s", l.Library, l.Version)
 }
@@ -378,6 +401,16 @@ func (l *ProfileLibraryReference) String() string {
 func (l *ProfileLibraryReference) InternalUniqueIdentifier() string {
 	f.Assert(l.InstallDir == nil,
 		"InternalUniqueIdentifier should not be called for library references with an install directory")
+
+	if l.GitURL != nil {
+		id := "git-" + utils.SanitizeName(l.GitURL.Host+l.GitURL.Path+"#"+l.GitURL.Fragment)
+		if len(id) > 50 {
+			id = id[:50]
+		}
+		h := sha256.Sum256([]byte(l.GitURL.String()))
+		return id + "-" + hex.EncodeToString(h[:])[:8]
+	}
+
 	id := l.String()
 	h := sha256.Sum256([]byte(id))
 	res := fmt.Sprintf("%s_%s", id, hex.EncodeToString(h[:])[:16])

internal/arduino/sketch/profiles_test.go

@@ -53,3 +53,26 @@ func TestProjectFileLoading(t *testing.T) {
 		require.Error(t, err)
 	}
 }
+
+func TestProjectFileLibraries(t *testing.T) {
+	sketchProj := paths.New("testdata", "profiles", "profile_with_libraries.yml")
+	proj, err := LoadProjectFile(sketchProj)
+	require.NoError(t, err)
+	require.Len(t, proj.Profiles, 1)
+	prof := proj.Profiles[0]
+	require.Len(t, prof.Libraries, 5)
+	require.Equal(t, "[email protected]", prof.Libraries[0].String())
+	require.Equal(t, "@dir:/path/to/system/lib", prof.Libraries[1].String())
+	require.Equal(t, "@dir:path/to/sketch/lib", prof.Libraries[2].String())
+	require.Equal(t, "@git:https://github.com/username/HelloWorld.git#v2.13", prof.Libraries[3].String())
+	require.Equal(t, "@git:https://github.com/username/HelloWorld.git#v2.14", prof.Libraries[4].String())
+	require.Equal(t, "FlashStorage_1.2.3_e525d7c96b27788f", prof.Libraries[0].InternalUniqueIdentifier())
+	require.Panics(t, func() { prof.Libraries[1].InternalUniqueIdentifier() })
+	require.Panics(t, func() { prof.Libraries[2].InternalUniqueIdentifier() })
+	require.Equal(t, "git-github.202132.xyz_username_HelloWorld.git_v2.13-0c146203", prof.Libraries[3].InternalUniqueIdentifier())
+	require.Equal(t, "git-github.202132.xyz_username_HelloWorld.git_v2.14-49f5df7f", prof.Libraries[4].InternalUniqueIdentifier())
+
+	orig, err := sketchProj.ReadFile()
+	require.NoError(t, err)
+	require.Equal(t, string(orig), proj.AsYaml())
+}

internal/arduino/sketch/testdata/profiles/profile_with_libraries.yml

@@ -0,0 +1,13 @@
+profiles:
+  giga:
+    fqbn: arduino:mbed_giga:giga
+    platforms:
+      - platform: arduino:mbed_giga (4.3.1)
+    libraries:
+      - FlashStorage (1.2.3)
+      - dir: /path/to/system/lib
+      - dir: path/to/sketch/lib
+      - git: https://github.com/username/HelloWorld.git#v2.13
+      - git: https://github.com/username/HelloWorld.git#v2.14
+
+default_profile: giga_any