hookscripts: glob hook directories

Move the directory searching / globbing code out of slurpFiles and into
the filelist module so it can be used elsewhere.

cmd/mkinitfs/main.go

@@ -47,6 +47,8 @@ func main() {
 		return
 	}
 
+	log.Default().SetFlags(log.Lmicroseconds)
+
 	deviceinfoFile := "/etc/deviceinfo"
 	if exists, err := misc.Exists(deviceinfoFile); !exists {
 		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
@@ -63,10 +65,6 @@ func main() {
 		return
 	}
 
-	// deviceinfo.InitfsCompression needs a little more post-processing
-	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	log.Printf("Using compression format %s with level %q\n", compressionFormat, compressionLevel)
-
 	defer misc.TimeFunc(time.Now(), "mkinitfs")
 
 	kernVer, err := osutil.GetKernelVersion()
@@ -95,13 +93,15 @@ func main() {
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
+	// deviceinfo.InitfsCompression needs a little more post-processing
+	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
 	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
-		hookscripts.New("/usr/share/mkinitfs/hooks"),
-		hookscripts.New("/etc/mkinitfs/hooks"),
+		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
+		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
 		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
 		modules.New([]string{}, "/etc/mkinitfs/modules"),
 	}); err != nil {
@@ -111,14 +111,13 @@ func main() {
 		return
 	}
 
-	// Note: compression disabled for initramfs-extra, since it slows down boot
-	// and can add more requirements to the initramfs (e.g. need to add support
-	// for extracting zstd)
-	if err := generateArchive("initramfs-extra", archive.FormatNone, archive.LevelDefault, workDir, []filelist.FileLister{
+	// deviceinfo.InitfsExtraCompression needs a little more post-processing
+	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
+	if err := generateArchive("initramfs-extra", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
 		hookfiles.New("/usr/share/mkinitfs/files-extra"),
 		hookfiles.New("/etc/mkinitfs/files-extra"),
-		hookscripts.New("/usr/share/mkinitfs/hooks-extra"),
-		hookscripts.New("/etc/mkinitfs/hooks-extra"),
+		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
+		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
 		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
 		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
 		osksdl.New(devinfo.MesaDriver),
@@ -150,11 +149,10 @@ func bootDeploy(workDir, outDir, ubootBoardname string) error {
 
 func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
 	log.Printf("== Generating %s ==\n", name)
+	log.Printf("- Using compression format %s with level %q\n", format, level)
+
 	defer misc.TimeFunc(time.Now(), name)
-	a, err := archive.New(format, level)
-	if err != nil {
-		return err
-	}
+	a := archive.New(format, level)
 
 	fs := initramfs.New(features)
 	if err := a.AddItems(fs); err != nil {

doc/mkinitfs.1.scd

@@ -24,6 +24,15 @@ is purely to generate the archive(s). mkinitfs does call *boot-deploy* after
 creating the archive(s), in order to install/deploy them and any other relevant
 boot-related items onto the system.
 
+Design goals of this project are:
+
+	- Support as many distros as possible
+	- Simplify configuration, while still giving multiple opportunities to set or override defaults
+	- Execute an external app to do any boot install/setup finalization
+		- One such app is here: https://gitlab.com/postmarketOS/boot-deploy
+		- But implementation can be anything, see the section on *BOOT-DEPLOY*
+		for more info
+
 # DEVICEINFO
 
 The canonical deviceinfo "specification" is at
@@ -33,6 +42,7 @@ mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
 are *required* by mkinitfs:
 
 	- deviceinfo_initfs_compression
+	- deviceinfo_initfs_extra_compression
 	- deviceinfo_mesa_driver
 	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
@@ -40,6 +50,10 @@ are *required* by mkinitfs:
 It is a design goal to keep the number of required variables from deviceinfo to
 a bare minimum, and to require only variables that don't hold lists of things.
 
+*NOTE*: When deviceinfo_initfs_extra_compression is set, make sure that the
+necessary tools to extract the configured archive format are in the initramfs
+archive.
+
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and
@@ -112,6 +126,30 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+# BOOT-DEPLOY
+
+After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
+search for the app. The following commandline options are passed to it:
+
+	*-i* <initramfs filename>
+
+		Currently this is hardcoded to be "initramfs"
+
+	*-k* <kernel filename>
+	
+	*-d* <work directory>
+
+		Path to the directory containing the build artifacts from mkinitfs.
+
+	*-o* <destination directory>
+
+		Path to the directory that boot-deploy should use as its root when
+		installing files.
+
+	*initramfs-extra*
+
+		This string is the filename of the initramfs-extra archive.
+
 # AUTHORS
 
 *Clayton Craft* <clayton@craftyguy.net>

internal/archive/archive.go

@@ -44,14 +44,14 @@ const (
 )
 
 type Archive struct {
-	items           archiveItems
 	cpioWriter      *cpio.Writer
 	buf             *bytes.Buffer
 	compress_format CompressFormat
 	compress_level  CompressLevel
+	items           archiveItems
 }
 
-func New(format CompressFormat, level CompressLevel) (*Archive, error) {
+func New(format CompressFormat, level CompressLevel) *Archive {
 	buf := new(bytes.Buffer)
 	archive := &Archive{
 		cpioWriter:      cpio.NewWriter(buf),
@@ -60,12 +60,12 @@ func New(format CompressFormat, level CompressLevel) (*Archive, error) {
 		compress_level:  level,
 	}
 
-	return archive, nil
+	return archive
 }
 
 type archiveItem struct {
-	sourcePath string
 	header     *cpio.Header
+	sourcePath string
 }
 
 type archiveItems struct {
@@ -184,10 +184,11 @@ func (archive *Archive) Write(path string, mode os.FileMode) error {
 	return nil
 }
 
-// Adds the given items in the map to the archive. The map format is {source path:dest path}.
-// Internally this just calls AddItem on each key,value pair in the map.
-func (archive *Archive) AddItems(f filelist.FileLister) error {
-	list, err := f.List()
+// AddItems adds the given items in the map to the archive. The map format is
+// {source path:dest path}. Internally this just calls AddItem on each
+// key,value pair in the map.
+func (archive *Archive) AddItems(flister filelist.FileLister) error {
+	list, err := flister.List()
 	if err != nil {
 		return err
 	}

internal/bootdeploy/bootdeploy.go

@@ -2,10 +2,12 @@ package bootdeploy
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -68,7 +70,8 @@ func bootDeploy(workDir string, outDir string) error {
 	}
 	defer kernFd.Close()
 
-	kernFileCopy, err := os.Create(filepath.Join(workDir, "vmlinuz"))
+	kernFilename := path.Base(kernFile)
+	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -76,12 +79,14 @@ func bootDeploy(workDir string, outDir string) error {
 	if _, err = io.Copy(kernFileCopy, kernFd); err != nil {
 		return err
 	}
-	kernFileCopy.Close()
+	if err := kernFileCopy.Close(); err != nil {
+		return fmt.Errorf("error closing %s: %w", kernFilename, err)
+	}
 
 	// boot-deploy -i initramfs -k vmlinuz-postmarketos-rockchip -d /tmp/cpio -o /tmp/foo initramfs-extra
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
-		"-k", "vmlinuz",
+		"-k", kernFilename,
 		"-d", workDir,
 		"-o", outDir,
 		"initramfs-extra")

internal/filelist/filelist.go

@@ -1,6 +1,12 @@
 package filelist
 
-import "sync"
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
+)
 
 type FileLister interface {
 	List() (*FileList, error)
@@ -45,6 +51,26 @@ func (f *FileList) Import(src *FileList) {
 	}
 }
 
+func (f *FileList) AddGlobbed(src string, dest string) error {
+	fFiles, err := misc.GetFiles([]string{src}, true)
+	if err != nil {
+		return fmt.Errorf("unable to add %q: %w", src, err)
+	}
+	// loop over all returned files from GetFile
+	for _, file := range fFiles {
+		if len(fFiles) > 1 {
+			// Glob with arbitrary subdirectories, so we need to
+			// remove the src path and prepend the dest path
+			f.Add(file, filepath.Join(dest, file[len(src):]))
+		} else {
+			// dest path specified, and only 1 file
+			f.Add(file, dest)
+		}
+	}
+
+	return nil
+}
+
 // iterate through the list and and send each one as a new File over the
 // returned channel
 func (f *FileList) IterItems() <-chan File {

internal/filelist/hookdirs/hookdirs.go

@@ -23,7 +23,7 @@ func New(path string) *HookDirs {
 }
 
 func (h *HookDirs) List() (*filelist.FileList, error) {
-	log.Printf("- Creating directories specified in %s", h.path)
+	log.Printf("- Searching for directories specified in %s", h.path)
 
 	files := filelist.NewFileList()
 	fileInfo, err := os.ReadDir(h.path)

internal/filelist/hookfiles/hookfiles.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type HookFiles struct {
@@ -26,7 +25,7 @@ func New(filePath string) *HookFiles {
 }
 
 func (h *HookFiles) List() (*filelist.FileList, error) {
-	log.Printf("- Including file lists from %s", h.filePath)
+	log.Printf("- Searching for file lists from %s", h.filePath)
 
 	files := filelist.NewFileList()
 	fileInfo, err := os.ReadDir(h.filePath)
@@ -60,22 +59,13 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 	for s.Scan() {
 		src, dest, has_dest := strings.Cut(s.Text(), ":")
 
-		fFiles, err := misc.GetFiles([]string{src}, true)
-		if err != nil {
-			return nil, fmt.Errorf("unable to add %q: %w", src, err)
+		if !has_dest {
+			dest = src
 		}
-		// loop over all returned files from GetFile
-		for _, file := range fFiles {
-			if !has_dest {
-				files.Add(file, file)
-			} else if len(fFiles) > 1 {
-				// Don't support specifying dest if src was a glob
-				// NOTE: this could support this later...
-				files.Add(file, file)
-			} else {
-				// dest path specified, and only 1 file
-				files.Add(file, dest)
-			}
+
+		err := files.AddGlobbed(src, dest)
+		if err != nil {
+			return nil, err
 		}
 	}
 

internal/filelist/hookscripts/hookscripts.go

@@ -9,19 +9,22 @@ import (
 )
 
 type HookScripts struct {
+	destPath   string
 	scriptsDir string
 }
 
 // New returns a new HookScripts that will use the given path to provide a list
-// of script files.
-func New(scriptsDir string) *HookScripts {
+// of script files. The destination for each script it set to destPath, using
+// the original file name.
+func New(scriptsDir string, destPath string) *HookScripts {
 	return &HookScripts{
+		destPath:   destPath,
 		scriptsDir: scriptsDir,
 	}
 }
 
 func (h *HookScripts) List() (*filelist.FileList, error) {
-	log.Printf("- Including hook scripts from %s", h.scriptsDir)
+	log.Printf("- Searching for hook scripts from %s", h.scriptsDir)
 
 	files := filelist.NewFileList()
 
@@ -32,7 +35,13 @@ func (h *HookScripts) List() (*filelist.FileList, error) {
 	}
 	for _, file := range fileInfo {
 		path := filepath.Join(h.scriptsDir, file.Name())
-		files.Add(path, path)
+		if file.IsDir() {
+			log.Printf("-- Including dir %s\n", path)
+			files.AddGlobbed(filepath.Join(path, "*"), filepath.Join(h.destPath, file.Name()))
+		} else {
+			log.Printf("-- Including script: %s\n", path)
+			files.Add(path, filepath.Join(h.destPath, file.Name()))
+		}
 	}
 	return files, nil
 }

internal/filelist/modules/modules.go

@@ -16,8 +16,8 @@ import (
 )
 
 type Modules struct {
-	modulesList     []string
 	modulesListPath string
+	modulesList     []string
 }
 
 // New returns a new Modules that will use the given moduleto provide a list
@@ -30,8 +30,6 @@ func New(modulesList []string, modulesListPath string) *Modules {
 }
 
 func (m *Modules) List() (*filelist.FileList, error) {
-	log.Println("- Including kernel modules")
-
 	kernVer, err := osutil.GetKernelVersion()
 	if err != nil {
 		return nil, err
@@ -55,20 +53,23 @@ func (m *Modules) List() (*filelist.FileList, error) {
 	}
 
 	// slurp up given list of modules
-	for _, module := range m.modulesList {
-		if modFilelist, err := getModule(module, modDir); err != nil {
-			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
-		} else {
-			for _, file := range modFilelist {
-				files.Add(file, file)
+	if len(m.modulesList) > 0 {
+		log.Printf("-- Including kernel modules from deviceinfo")
+		for _, module := range m.modulesList {
+			if modFilelist, err := getModule(module, modDir); err != nil {
+				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
+			} else {
+				for _, file := range modFilelist {
+					files.Add(file, file)
+				}
 			}
 		}
 	}
 
 	// slurp up modules from lists in modulesListPath
+	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
 	if err != nil {
-		log.Println("-- Unable to find dir, skipping...")
 		return files, nil
 	}
 	for _, file := range fileInfo {
@@ -126,7 +127,11 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 }
 
 func getModulesInDir(modPath string) (files []string, err error) {
-	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
+	err = filepath.Walk(modPath, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			// Unable to walk path
+			return err
+		}
 		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
 			return nil
 		}

internal/misc/misc.go

@@ -45,7 +45,7 @@ func RemoveDuplicates(in []string) (out []string) {
 //	defer misc.TimeFunc(time.Now(), "foo")
 func TimeFunc(start time.Time, name string) {
 	elapsed := time.Since(start)
-	log.Printf("%s completed in: %s", name, elapsed)
+	log.Printf("%s completed in: %.2fs", name, elapsed.Seconds())
 }
 
 // Exists tests if the given file/dir exists or not. Returns any errors related

pkgs/deviceinfo/deviceinfo.go

@@ -14,10 +14,11 @@ import (
 )
 
 type DeviceInfo struct {
-	InitfsCompression string
-	MesaDriver        string
-	ModulesInitfs     string
-	UbootBoardname    string
+	InitfsCompression      string
+	InitfsExtraCompression string
+	MesaDriver             string
+	ModulesInitfs          string
+	UbootBoardname         string
 }
 
 func ReadDeviceinfo(file string) (DeviceInfo, error) {

pkgs/deviceinfo/deviceinfo_test.go

@@ -21,6 +21,7 @@ func TestNameToField(t *testing.T) {
 		{"deviceinfo_modules_initfs", "ModulesInitfs"},
 		{"modules_initfs", "ModulesInitfs"},
 		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
 	for _, table := range tables {