hookscripts: glob hook directories

Move the directory searching / globbing code out of slurpFiles and into
the filelist module so it can be used elsewhere.

cmd/mkinitfs/main.go

@@ -152,10 +152,7 @@ func generateArchive(name string, format archive.CompressFormat, level archive.C
 	log.Printf("- Using compression format %s with level %q\n", format, level)
 
 	defer misc.TimeFunc(time.Now(), name)
-	a, err := archive.New(format, level)
-	if err != nil {
-		return err
-	}
+	a := archive.New(format, level)
 
 	fs := initramfs.New(features)
 	if err := a.AddItems(fs); err != nil {

doc/mkinitfs.1.scd

@@ -24,6 +24,15 @@ is purely to generate the archive(s). mkinitfs does call *boot-deploy* after
 creating the archive(s), in order to install/deploy them and any other relevant
 boot-related items onto the system.
 
+Design goals of this project are:
+
+	- Support as many distros as possible
+	- Simplify configuration, while still giving multiple opportunities to set or override defaults
+	- Execute an external app to do any boot install/setup finalization
+		- One such app is here: https://gitlab.com/postmarketOS/boot-deploy
+		- But implementation can be anything, see the section on *BOOT-DEPLOY*
+		for more info
+
 # DEVICEINFO
 
 The canonical deviceinfo "specification" is at
@@ -117,6 +126,30 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+# BOOT-DEPLOY
+
+After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
+search for the app. The following commandline options are passed to it:
+
+	*-i* <initramfs filename>
+
+		Currently this is hardcoded to be "initramfs"
+
+	*-k* <kernel filename>
+	
+	*-d* <work directory>
+
+		Path to the directory containing the build artifacts from mkinitfs.
+
+	*-o* <destination directory>
+
+		Path to the directory that boot-deploy should use as its root when
+		installing files.
+
+	*initramfs-extra*
+
+		This string is the filename of the initramfs-extra archive.
+
 # AUTHORS
 
 *Clayton Craft* <clayton@craftyguy.net>

internal/archive/archive.go

@@ -44,14 +44,14 @@ const (
 )
 
 type Archive struct {
-	items           archiveItems
 	cpioWriter      *cpio.Writer
 	buf             *bytes.Buffer
 	compress_format CompressFormat
 	compress_level  CompressLevel
+	items           archiveItems
 }
 
-func New(format CompressFormat, level CompressLevel) (*Archive, error) {
+func New(format CompressFormat, level CompressLevel) *Archive {
 	buf := new(bytes.Buffer)
 	archive := &Archive{
 		cpioWriter:      cpio.NewWriter(buf),
@@ -60,12 +60,12 @@ func New(format CompressFormat, level CompressLevel) (*Archive, error) {
 		compress_level:  level,
 	}
 
-	return archive, nil
+	return archive
 }
 
 type archiveItem struct {
-	sourcePath string
 	header     *cpio.Header
+	sourcePath string
 }
 
 type archiveItems struct {
@@ -184,10 +184,11 @@ func (archive *Archive) Write(path string, mode os.FileMode) error {
 	return nil
 }
 
-// Adds the given items in the map to the archive. The map format is {source path:dest path}.
-// Internally this just calls AddItem on each key,value pair in the map.
-func (archive *Archive) AddItems(f filelist.FileLister) error {
-	list, err := f.List()
+// AddItems adds the given items in the map to the archive. The map format is
+// {source path:dest path}. Internally this just calls AddItem on each
+// key,value pair in the map.
+func (archive *Archive) AddItems(flister filelist.FileLister) error {
+	list, err := flister.List()
 	if err != nil {
 		return err
 	}

internal/filelist/filelist.go

@@ -1,6 +1,12 @@
 package filelist
 
-import "sync"
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
+)
 
 type FileLister interface {
 	List() (*FileList, error)
@@ -45,6 +51,26 @@ func (f *FileList) Import(src *FileList) {
 	}
 }
 
+func (f *FileList) AddGlobbed(src string, dest string) error {
+	fFiles, err := misc.GetFiles([]string{src}, true)
+	if err != nil {
+		return fmt.Errorf("unable to add %q: %w", src, err)
+	}
+	// loop over all returned files from GetFile
+	for _, file := range fFiles {
+		if len(fFiles) > 1 {
+			// Glob with arbitrary subdirectories, so we need to
+			// remove the src path and prepend the dest path
+			f.Add(file, filepath.Join(dest, file[len(src):]))
+		} else {
+			// dest path specified, and only 1 file
+			f.Add(file, dest)
+		}
+	}
+
+	return nil
+}
+
 // iterate through the list and and send each one as a new File over the
 // returned channel
 func (f *FileList) IterItems() <-chan File {

internal/filelist/hookfiles/hookfiles.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type HookFiles struct {
@@ -60,22 +59,13 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 	for s.Scan() {
 		src, dest, has_dest := strings.Cut(s.Text(), ":")
 
-		fFiles, err := misc.GetFiles([]string{src}, true)
-		if err != nil {
-			return nil, fmt.Errorf("unable to add %q: %w", src, err)
+		if !has_dest {
+			dest = src
 		}
-		// loop over all returned files from GetFile
-		for _, file := range fFiles {
-			if !has_dest {
-				files.Add(file, file)
-			} else if len(fFiles) > 1 {
-				// Don't support specifying dest if src was a glob
-				// NOTE: this could support this later...
-				files.Add(file, file)
-			} else {
-				// dest path specified, and only 1 file
-				files.Add(file, dest)
-			}
+
+		err := files.AddGlobbed(src, dest)
+		if err != nil {
+			return nil, err
 		}
 	}
 

internal/filelist/hookscripts/hookscripts.go

@@ -35,8 +35,13 @@ func (h *HookScripts) List() (*filelist.FileList, error) {
 	}
 	for _, file := range fileInfo {
 		path := filepath.Join(h.scriptsDir, file.Name())
-		log.Printf("-- Including script: %s\n", path)
-		files.Add(path, filepath.Join(h.destPath, file.Name()))
+		if file.IsDir() {
+			log.Printf("-- Including dir %s\n", path)
+			files.AddGlobbed(filepath.Join(path, "*"), filepath.Join(h.destPath, file.Name()))
+		} else {
+			log.Printf("-- Including script: %s\n", path)
+			files.Add(path, filepath.Join(h.destPath, file.Name()))
+		}
 	}
 	return files, nil
 }

internal/filelist/modules/modules.go

@@ -16,8 +16,8 @@ import (
 )
 
 type Modules struct {
-	modulesList     []string
 	modulesListPath string
+	modulesList     []string
 }
 
 // New returns a new Modules that will use the given moduleto provide a list
@@ -30,8 +30,6 @@ func New(modulesList []string, modulesListPath string) *Modules {
 }
 
 func (m *Modules) List() (*filelist.FileList, error) {
-	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
-
 	kernVer, err := osutil.GetKernelVersion()
 	if err != nil {
 		return nil, err
@@ -55,20 +53,23 @@ func (m *Modules) List() (*filelist.FileList, error) {
 	}
 
 	// slurp up given list of modules
-	for _, module := range m.modulesList {
-		if modFilelist, err := getModule(module, modDir); err != nil {
-			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
-		} else {
-			for _, file := range modFilelist {
-				files.Add(file, file)
+	if len(m.modulesList) > 0 {
+		log.Printf("-- Including kernel modules from deviceinfo")
+		for _, module := range m.modulesList {
+			if modFilelist, err := getModule(module, modDir); err != nil {
+				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
+			} else {
+				for _, file := range modFilelist {
+					files.Add(file, file)
+				}
 			}
 		}
 	}
 
 	// slurp up modules from lists in modulesListPath
+	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
 	if err != nil {
-		log.Println("-- Unable to find dir, skipping...")
 		return files, nil
 	}
 	for _, file := range fileInfo {
@@ -126,7 +127,11 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 }
 
 func getModulesInDir(modPath string) (files []string, err error) {
-	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
+	err = filepath.Walk(modPath, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			// Unable to walk path
+			return err
+		}
 		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
 			return nil
 		}

internal/misc/misc.go

@@ -45,7 +45,7 @@ func RemoveDuplicates(in []string) (out []string) {
 //	defer misc.TimeFunc(time.Now(), "foo")
 func TimeFunc(start time.Time, name string) {
 	elapsed := time.Since(start)
-	log.Printf("%s completed in: %s", name, elapsed)
+	log.Printf("%s completed in: %.2fs", name, elapsed.Seconds())
 }
 
 // Exists tests if the given file/dir exists or not. Returns any errors related