Add --library flag to compile command (#1258)

* Add --library flag to compile command

* Fix library prioritization

* [skip changelog] Fix receiver name in LibraryManager functions

* [skip changelog] Fix variables names and some docstrings

* Fix libraries not being recompiled when path to source file changes

Author: silvanocerza

arduino/libraries/libraries_location.go

@@ -35,6 +35,9 @@ const (
 	ReferencedPlatformBuiltIn
 	// User are user installed libraries
 	User
+	// Unmanaged is for libraries set manually by the user in the CLI command or from the gRPC function.
+	// Ideally it's used for `libraries` outside folders managed by the CLI.
+	Unmanaged
 )
 
 func (d *LibraryLocation) String() string {
@@ -47,6 +50,8 @@ func (d *LibraryLocation) String() string {
 		return "ref-platform"
 	case User:
 		return "user"
+	case Unmanaged:
+		return "unmanaged"
 	}
 	panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 }
@@ -62,6 +67,8 @@ func (d *LibraryLocation) MarshalJSON() ([]byte, error) {
 		return json.Marshal("ref-platform")
 	case User:
 		return json.Marshal("user")
+	case Unmanaged:
+		return json.Marshal("unmanaged")
 	}
 	return nil, fmt.Errorf("invalid library location value: %d", *d)
 }
@@ -81,6 +88,8 @@ func (d *LibraryLocation) UnmarshalJSON(b []byte) error {
 		*d = ReferencedPlatformBuiltIn
 	case "user":
 		*d = User
+	case "unmanaged":
+		*d = Unmanaged
 	}
 	return fmt.Errorf("invalid library location: %s", s)
 }
@@ -96,6 +105,8 @@ func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation {
 		return rpc.LibraryLocation_LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN
 	case User:
 		return rpc.LibraryLocation_LIBRARY_LOCATION_USER
+	case Unmanaged:
+		return rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED
 	}
 	panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 }
@@ -111,6 +122,8 @@ func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation {
 		return ReferencedPlatformBuiltIn
 	case rpc.LibraryLocation_LIBRARY_LOCATION_USER:
 		return User
+	case rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED:
+		return Unmanaged
 	}
 	panic(fmt.Sprintf("invalid rpc.LibraryLocation value %d", l))
 }

arduino/libraries/librariesmanager/librariesmanager.go

@@ -82,10 +82,10 @@ func (alts *LibraryAlternatives) FindVersion(version *semver.Version) *libraries
 }
 
 // Names returns an array with all the names of the installed libraries.
-func (sc LibrariesManager) Names() []string {
-	res := make([]string, len(sc.Libraries))
+func (lm LibrariesManager) Names() []string {
+	res := make([]string, len(lm.Libraries))
 	i := 0
-	for n := range sc.Libraries {
+	for n := range lm.Libraries {
 		res[i] = n
 		i++
 	}
@@ -109,27 +109,27 @@ func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *Librarie
 
 // LoadIndex reads a library_index.json from a file and returns
 // the corresponding Index structure.
-func (sc *LibrariesManager) LoadIndex() error {
-	index, err := librariesindex.LoadIndex(sc.IndexFile)
+func (lm *LibrariesManager) LoadIndex() error {
+	index, err := librariesindex.LoadIndex(lm.IndexFile)
 	if err != nil {
-		sc.Index = librariesindex.EmptyIndex
+		lm.Index = librariesindex.EmptyIndex
 		return err
 	}
-	sc.Index = index
+	lm.Index = index
 	return nil
 }
 
 // AddLibrariesDir adds path to the list of directories
 // to scan when searching for libraries. If a path is already
 // in the list it is ignored.
-func (sc *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries.LibraryLocation) {
-	for _, dir := range sc.LibrariesDir {
+func (lm *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries.LibraryLocation) {
+	for _, dir := range lm.LibrariesDir {
 		if dir.Path.EquivalentTo(path) {
 			return
 		}
 	}
 	logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir")
-	sc.LibrariesDir = append(sc.LibrariesDir, &LibrariesDir{
+	lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{
 		Path:     path,
 		Location: location,
 	})
@@ -138,36 +138,36 @@ func (sc *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries
 // AddPlatformReleaseLibrariesDir add the libraries directory in the
 // specified PlatformRelease to the list of directories to scan when
 // searching for libraries.
-func (sc *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *cores.PlatformRelease, location libraries.LibraryLocation) {
+func (lm *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *cores.PlatformRelease, location libraries.LibraryLocation) {
 	path := plaftormRelease.GetLibrariesDir()
 	if path == nil {
 		return
 	}
-	for _, dir := range sc.LibrariesDir {
+	for _, dir := range lm.LibrariesDir {
 		if dir.Path.EquivalentTo(path) {
 			return
 		}
 	}
 	logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir")
-	sc.LibrariesDir = append(sc.LibrariesDir, &LibrariesDir{
+	lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{
 		Path:            path,
 		Location:        location,
 		PlatformRelease: plaftormRelease,
 	})
 }
 
 // RescanLibraries reload all installed libraries in the system.
-func (sc *LibrariesManager) RescanLibraries() error {
-	for _, dir := range sc.LibrariesDir {
-		if err := sc.LoadLibrariesFromDir(dir); err != nil {
+func (lm *LibrariesManager) RescanLibraries() error {
+	for _, dir := range lm.LibrariesDir {
+		if err := lm.LoadLibrariesFromDir(dir); err != nil {
 			return fmt.Errorf("loading libs from %s: %s", dir.Path, err)
 		}
 	}
 	return nil
 }
 
-func (sc *LibrariesManager) getUserLibrariesDir() *paths.Path {
-	for _, dir := range sc.LibrariesDir {
+func (lm *LibrariesManager) getUserLibrariesDir() *paths.Path {
+	for _, dir := range lm.LibrariesDir {
 		if dir.Location == libraries.User {
 			return dir.Path
 		}
@@ -177,7 +177,7 @@ func (sc *LibrariesManager) getUserLibrariesDir() *paths.Path {
 
 // LoadLibrariesFromDir loads all libraries in the given directory. Returns
 // nil if the directory doesn't exists.
-func (sc *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) error {
+func (lm *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) error {
 	subDirs, err := librariesDir.Path.ReadDir()
 	if os.IsNotExist(err) {
 		return nil
@@ -194,22 +194,45 @@ func (sc *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) err
 			return fmt.Errorf("loading library from %s: %s", subDir, err)
 		}
 		library.ContainerPlatform = librariesDir.PlatformRelease
-		alternatives, ok := sc.Libraries[library.Name]
+		alternatives, ok := lm.Libraries[library.Name]
 		if !ok {
 			alternatives = &LibraryAlternatives{}
-			sc.Libraries[library.Name] = alternatives
+			lm.Libraries[library.Name] = alternatives
 		}
 		alternatives.Add(library)
 	}
 	return nil
 }
 
+// LoadLibraryFromDir loads one single library from the libRootDir.
+// libRootDir must point to the root of a valid library.
+// An error is returned if the path doesn't exist or loading of the library fails.
+func (lm *LibrariesManager) LoadLibraryFromDir(libRootDir *paths.Path, location libraries.LibraryLocation) error {
+	if libRootDir.NotExist() {
+		return fmt.Errorf("library path does not exist: %s", libRootDir)
+	}
+
+	library, err := libraries.Load(libRootDir, location)
+	if err != nil {
+		return fmt.Errorf("loading library from %s: %s", libRootDir, err)
+	}
+
+	alternatives, ok := lm.Libraries[library.Name]
+	if !ok {
+		alternatives = &LibraryAlternatives{}
+		lm.Libraries[library.Name] = alternatives
+	}
+	alternatives.Add(library)
+
+	return nil
+}
+
 // FindByReference return the installed library matching the Reference
 // name and version or, if the version is nil, the library installed
 // in the User folder.
-func (sc *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *libraries.Library {
+func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *libraries.Library {
 	saneName := utils.SanitizeName(libRef.Name)
-	alternatives, have := sc.Libraries[saneName]
+	alternatives, have := lm.Libraries[saneName]
 	if !have {
 		return nil
 	}

arduino/libraries/librariesresolver/cpp.go

@@ -158,6 +158,8 @@ func computePriority(lib *libraries.Library, header, arch string) int {
 		priority += 2
 	case libraries.User:
 		priority += 3
+	case libraries.Unmanaged:
+		priority += 4
 	default:
 		panic(fmt.Sprintf("Invalid library location: %d", lib.Location))
 	}

cli/compile/compile.go

@@ -51,12 +51,16 @@ var (
 	port                    string   // Upload port, e.g.: COM10 or /dev/ttyACM0.
 	verify                  bool     // Upload, verify uploaded binary after the upload.
 	exportDir               string   // The compiled binary is written to this file
-	libraries               []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
 	optimizeForDebug        bool     // Optimize compile output for debug, not for release
 	programmer              string   // Use the specified programmer to upload
 	clean                   bool     // Cleanup the build folder and do not use any cached build
 	compilationDatabaseOnly bool     // Only create compilation database without actually compiling
 	sourceOverrides         string   // Path to a .json file that contains a set of replacements of the sketch source code.
+	// library and libraries sound similar but they're actually different.
+	// library expects a path to the root folder of one single library.
+	// libraries expects a path to a directory containing multiple libraries, similarly to the <directories.user>/libraries path.
+	library   []string // List of paths to libraries root folders. Can be used multiple times for different libraries
+	libraries []string // List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries paths.
 )
 
 // NewCommand created a new `compile` command
@@ -93,8 +97,10 @@ func NewCommand() *cobra.Command {
 	command.Flags().StringVarP(&port, "port", "p", "", "Upload port, e.g.: COM10 or /dev/ttyACM0")
 	command.Flags().BoolVarP(&verify, "verify", "t", false, "Verify uploaded binary after the upload.")
 	command.Flags().StringVar(&vidPid, "vid-pid", "", "When specified, VID/PID specific build properties are used, if board supports them.")
+	command.Flags().StringSliceVar(&library, "library", []string{},
+		"List of paths to libraries root folders. Libraries set this way have top priority in case of conflicts. Can be used multiple times for different libraries.")
 	command.Flags().StringSliceVar(&libraries, "libraries", []string{},
-		"List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.")
+		"List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries dir paths.")
 	command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.")
 	command.Flags().StringVarP(&programmer, "programmer", "P", "", "Optional, use the specified programmer to upload.")
 	command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, "Just produce the compilation database, without actually compiling.")
@@ -171,6 +177,7 @@ func run(cmd *cobra.Command, args []string) {
 		Clean:                         clean,
 		CreateCompilationDatabaseOnly: compilationDatabaseOnly,
 		SourceOverride:                overrides,
+		Library:                       library,
 	}
 	compileOut := new(bytes.Buffer)
 	compileErr := new(bytes.Buffer)

commands/compile/compile.go

@@ -137,6 +137,8 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...)
 	builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings))
 
+	builderCtx.LibraryDirs = paths.NewPathList(req.Library...)
+
 	if req.GetBuildPath() == "" {
 		builderCtx.BuildPath = bldr.GenBuildPath(sketch.FullPath)
 	} else {

docs/sketch-build-process.md

@@ -102,6 +102,8 @@ The "folder name priority" is determined as follows (in order of highest to lowe
 
 The "location priority" is determined as follows (in order of highest to lowest priority):
 
+1. The library is specified using the [`--library` option](commands/arduino-cli_compile.md#options) of
+   `arduino-cli compile`
 1. The library is under a custom libraries path specified via the
    [`--libraries` option](commands/arduino-cli_compile.md#options) of `arduino-cli compile` (in decreasing order of
    priority when multiple custom paths are defined)

legacy/builder/builder_utils/utils.go

@@ -357,6 +357,16 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil
 		return false, nil
 	}
 
+	// The first line of the depfile contains the path to the object file to generate.
+	// The second line of the depfile contains the path to the source file.
+	// All subsequent lines contain the header files necessary to compile the object file.
+
+	// If we don't do this check it might happen that trying to compile a source file
+	// that has the same name but a different path wouldn't recreate the object file.
+	if sourceFile.String() != strings.Trim(rows[1], " ") {
+		return false, nil
+	}
+
 	rows = rows[1:]
 	for _, row := range rows {
 		depStat, err := os.Stat(row)

legacy/builder/libraries_loader.go

@@ -61,6 +61,13 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error {
 		return errors.WithStack(err)
 	}
 
+	for _, dir := range ctx.LibraryDirs {
+		// Libraries specified this way have top priority
+		if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil {
+			return err
+		}
+	}
+
 	if debugLevel > 0 {
 		for _, lib := range lm.Libraries {
 			for _, libAlt := range lib.Alternatives {

legacy/builder/types/context.go

@@ -67,6 +67,7 @@ type Context struct {
 	BuiltInToolsDirs     paths.PathList
 	BuiltInLibrariesDirs paths.PathList
 	OtherLibrariesDirs   paths.PathList
+	LibraryDirs          paths.PathList // List of paths pointing to individual library root folders
 	SketchLocation       *paths.Path
 	WatchedLocations     paths.PathList
 	ArduinoAPIVersion    string