hookscripts: glob hook directories

Move the directory searching / globbing code out of slurpFiles and into
the filelist module so it can be used elsewhere.

cmd/mkinitfs/main.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/archive"
@@ -19,6 +20,7 @@ import (
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/hookscripts"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/initramfs"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
@@ -47,13 +49,18 @@ func main() {
 
 	log.Default().SetFlags(log.Lmicroseconds)
 
-	var devinfo deviceinfo.DeviceInfo
-	deverr_usr := devinfo.ReadDeviceinfo("/usr/share/deviceinfo/deviceinfo")
-	deverr_etc := devinfo.ReadDeviceinfo("/etc/deviceinfo")
-	if deverr_etc != nil && deverr_usr != nil {
-		log.Println("Error reading deviceinfo")
-		log.Println("\t/usr/share/deviceinfo/deviceinfo:", deverr_usr)
-		log.Println("\t/etc/deviceinfo:", deverr_etc)
+	deviceinfoFile := "/etc/deviceinfo"
+	if exists, err := misc.Exists(deviceinfoFile); !exists {
+		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
+		return
+	} else if err != nil {
+		retCode = 1
+		log.Printf("received unexpected error when getting status for %q: %s", deviceinfoFile, err)
+	}
+
+	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
+	if err != nil {
+		log.Println(err)
 		retCode = 1
 		return
 	}
@@ -78,83 +85,52 @@ func main() {
 	defer func() {
 		e := os.RemoveAll(workDir)
 		if e != nil && err == nil {
-			log.Println(e)
-			log.Println("unable to remove temporary work directory")
+			err = e
+			retCode = 1
 		}
 	}()
 
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
-	//
-	// initramfs
-	//
 	// deviceinfo.InitfsCompression needs a little more post-processing
 	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	log.Printf("== Generating %s ==\n", "initramfs")
-	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
-
-	start := time.Now()
-	initramfsAr := archive.New(compressionFormat, compressionLevel)
-	initfs := initramfs.New([]filelist.FileLister{
+	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
 		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
 		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
-		modules.New("/usr/share/mkinitfs/modules"),
-		modules.New("/etc/mkinitfs/modules"),
-	})
-	if err := initramfsAr.AddItems(initfs); err != nil {
+		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
+		modules.New([]string{}, "/etc/mkinitfs/modules"),
+	}); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
-	if err := initramfsAr.Write(filepath.Join(workDir, "initramfs"), os.FileMode(0644)); err != nil {
-		log.Println(err)
-		log.Println("failed to generate: ", "initramfs")
-		retCode = 1
-		return
-	}
-	misc.TimeFunc(start, "initramfs")
 
-	//
-	// initramfs-extra
-	//
 	// deviceinfo.InitfsExtraCompression needs a little more post-processing
 	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
-	log.Printf("== Generating %s ==\n", "initramfs-extra")
-	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
-
-	start = time.Now()
-	initramfsExtraAr := archive.New(compressionFormat, compressionLevel)
-	initfsExtra := initramfs.New([]filelist.FileLister{
+	if err := generateArchive("initramfs-extra", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
 		hookfiles.New("/usr/share/mkinitfs/files-extra"),
 		hookfiles.New("/etc/mkinitfs/files-extra"),
 		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
 		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
-		modules.New("/usr/share/mkinitfs/modules-extra"),
-		modules.New("/etc/mkinitfs/modules-extra"),
-	})
-	if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
+		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
+		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
+		osksdl.New(devinfo.MesaDriver),
+	}); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs-extra")
 		retCode = 1
 		return
 	}
-	if err := initramfsExtraAr.Write(filepath.Join(workDir, "initramfs-extra"), os.FileMode(0644)); err != nil {
-		log.Println(err)
-		log.Println("failed to generate: ", "initramfs-extra")
-		retCode = 1
-		return
-	}
-	misc.TimeFunc(start, "initramfs-extra")
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
-		if err := bootDeploy(workDir, *outDir, devinfo); err != nil {
+		if err := bootDeploy(workDir, *outDir, devinfo.UbootBoardname); err != nil {
 			log.Println(err)
 			log.Println("boot-deploy failed")
 			retCode = 1
@@ -163,10 +139,30 @@ func main() {
 	}
 }
 
-func bootDeploy(workDir string, outDir string, devinfo deviceinfo.DeviceInfo) error {
+func bootDeploy(workDir, outDir, ubootBoardname string) error {
 	log.Print("== Using boot-deploy to finalize/install files ==")
 	defer misc.TimeFunc(time.Now(), "boot-deploy")
 
-	bd := bootdeploy.New(workDir, outDir, devinfo)
+	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
 	return bd.Run()
 }
+
+func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
+	log.Printf("== Generating %s ==\n", name)
+	log.Printf("- Using compression format %s with level %q\n", format, level)
+
+	defer misc.TimeFunc(time.Now(), name)
+	a := archive.New(format, level)
+
+	fs := initramfs.New(features)
+	if err := a.AddItems(fs); err != nil {
+		return err
+	}
+
+	log.Println("- Writing and verifying archive: ", name)
+	if err := a.Write(filepath.Join(path, name), os.FileMode(0644)); err != nil {
+		return err
+	}
+
+	return nil
+}

doc/mkinitfs.1.scd

@@ -38,13 +38,13 @@ Design goals of this project are:
 The canonical deviceinfo "specification" is at
 https://wiki.postmarketos.org/wiki/Deviceinfo_reference
 
-mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
-*/etc/deviceinfo*, in that order. The following variables
+mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
 are *required* by mkinitfs:
 
-	- deviceinfo_generate_systemd_boot
 	- deviceinfo_initfs_compression
 	- deviceinfo_initfs_extra_compression
+	- deviceinfo_mesa_driver
+	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
 
 It is a design goal to keep the number of required variables from deviceinfo to
@@ -54,36 +54,6 @@ a bare minimum, and to require only variables that don't hold lists of things.
 necessary tools to extract the configured archive format are in the initramfs
 archive.
 
-# ARCHIVE COMPRESSION
-
-Archive compression parameters are specified in the
-*deviceinfo_initfs_compression* and *deviceinfo_initfs_extra_compression*
-deviceinfo variables. Their values do not have to match, but special
-consideration should be taken since some formats may require additional kernel
-options or tools in the initramfs to support it.
-
-Supported compression *formats* for mkinitfs are:
-
-	- gzip
-	- lz4
-	- lzma
-	- none
-	- zstd
-
-Supported compression *levels* for mkinitfs:
-
-	- best
-	- default
-	- fast
-
-The value of these variables follows this syntax: *<format>:<level>*. For
-example, *zstd* with the *fast* compression level would be:
-*deviceinfo_initfs_compression="zstd:fast"*
-
-Defaults to *gzip* and *default* for both archives if format and/or level is
-unsupported or omitted.
-
-
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and
@@ -93,7 +63,7 @@ it are for constructing the initramfs archive.
 
 Configuration under */usr/share/mkinitfs* is intended to be managed by
 distributions, while configuration under */etc/mkinitfs* is for users to
-create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*.
+create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*. 
 
 ## /usr/share/mkinitfs/files, /etc/mkinitfs/files
 ## /usr/share/mkinitfs/files-extra, /etc/mkinitfs/files-extra
@@ -129,9 +99,6 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	path(s) under the relevant directory in */etc/mkinitfs*, and changing
 	the destination path.
 
-	Any lines in these files that start with *#* are considered comments, and
-	skipped.
-
 ## /usr/share/mkinitfs/hooks, /etc/mkinitfs/hooks
 ## /usr/share/mkinitfs/hooks-extra*, /etc/mkinitfs/hooks-extra
 
@@ -153,18 +120,12 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	Modules are installed in the initramfs archive under the same path they
 	exist on the system where mkinitfs is executed.
 
-	Any lines in these files that start with *#* are considered comments, and
-	skipped.
-
 ## /usr/share/mkinitfs/dirs, /etc/mkinitfs/dirs
 
 	Files with the *.dirs* extension in these directories are lists of
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
-	Any lines in these files that start with *#* are considered comments, and
-	skipped.
-
 # BOOT-DEPLOY
 
 After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
@@ -175,7 +136,7 @@ search for the app. The following commandline options are passed to it:
 		Currently this is hardcoded to be "initramfs"
 
 	*-k* <kernel filename>
-
+	
 	*-d* <work directory>
 
 		Path to the directory containing the build artifacts from mkinitfs.

go.mod

@@ -5,7 +5,6 @@ go 1.20
 require (
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.15.12
-	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/ulikunitz/xz v0.5.10
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 )

go.sum

@@ -2,8 +2,6 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RS
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
 github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
 github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
-github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
-github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=

internal/archive/archive.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/cavaliercoder/go-cpio"
 	"github.com/klauspost/compress/zstd"
-	"github.com/pierrec/lz4/v4"
 	"github.com/ulikunitz/xz"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
@@ -29,7 +28,6 @@ type CompressFormat string
 const (
 	FormatGzip CompressFormat = "gzip"
 	FormatLzma CompressFormat = "lzma"
-	FormatLz4  CompressFormat = "lz4"
 	FormatZstd CompressFormat = "zstd"
 	FormatNone CompressFormat = "none"
 )
@@ -106,7 +104,6 @@ func ExtractFormatLevel(s string) (format CompressFormat, level CompressLevel) {
 	case FormatLzma:
 		log.Println("Format lzma doesn't support a compression level, using default settings")
 		level = LevelDefault
-	case FormatLz4:
 	case FormatNone:
 	case FormatZstd:
 	default:
@@ -203,38 +200,6 @@ func (archive *Archive) AddItems(flister filelist.FileLister) error {
 	return nil
 }
 
-// AddItemsExclude is like AddItems, but takes a second FileLister that lists
-// items that should not be added to the archive from the first FileLister
-func (archive *Archive) AddItemsExclude(flister filelist.FileLister, exclude filelist.FileLister) error {
-	list, err := flister.List()
-	if err != nil {
-		return err
-	}
-
-	excludeList, err := exclude.List()
-	if err != nil {
-		return err
-	}
-
-	for i := range list.IterItems() {
-		dest, found := excludeList.Get(i.Source)
-
-		if found {
-			if i.Dest != dest {
-				found = false
-			}
-		}
-
-		if !found {
-			if err := archive.AddItem(i.Source, i.Dest); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 // Adds the given file or directory at "source" to the archive at "dest"
 func (archive *Archive) AddItem(source string, dest string) error {
 
@@ -351,23 +316,6 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 		if err != nil {
 			return err
 		}
-	case FormatLz4:
-		// The default compression for the lz4 library is Fast, and
-		// they don't define a Default level otherwise
-		level := lz4.Fast
-		switch archive.compress_level {
-		case LevelBest:
-			level = lz4.Level9
-		case LevelFast:
-			level = lz4.Fast
-		}
-
-		var writer = lz4.NewWriter(fd)
-		err = writer.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(level))
-		if err != nil {
-			return err
-		}
-		compressor = writer
 	case FormatNone:
 		compressor = fd
 	case FormatZstd:

internal/archive/archive_test.go

@@ -249,12 +249,6 @@ func TestExtractFormatLevel(t *testing.T) {
 			expectedFormat: FormatLzma,
 			expectedLevel:  LevelDefault,
 		},
-		{
-			name:           "lz4, fast",
-			in:             "lz4:fast",
-			expectedFormat: FormatLz4,
-			expectedLevel:  LevelFast,
-		},
 		{
 			name:           "none",
 			in:             "none",

internal/bootdeploy/bootdeploy.go

@@ -10,32 +10,32 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
 
 type BootDeploy struct {
-	inDir   string
-	outDir  string
-	devinfo deviceinfo.DeviceInfo
+	inDir          string
+	outDir         string
+	ubootBoardname string
 }
 
 // New returns a new BootDeploy, which then runs:
 //
 //	boot-deploy -d indir -o outDir
 //
-// devinfo is used to access some deviceinfo values, such as UbootBoardname
-// and GenerateSystemdBoot
-func New(inDir string, outDir string, devinfo deviceinfo.DeviceInfo) *BootDeploy {
+// ubootBoardname is used for copying in some u-boot files prior to running
+// boot-deploy. This is optional, passing an empty string is ok if this is not
+// needed.
+func New(inDir, outDir, ubootBoardname string) *BootDeploy {
 	return &BootDeploy{
-		inDir:   inDir,
-		outDir:  outDir,
-		devinfo: devinfo,
+		inDir:          inDir,
+		outDir:         outDir,
+		ubootBoardname: ubootBoardname,
 	}
 }
 
 func (b *BootDeploy) Run() error {
-	if err := copyUbootFiles(b.inDir, b.devinfo.UbootBoardname); errors.Is(err, os.ErrNotExist) {
+
+	if err := copyUbootFiles(b.inDir, b.ubootBoardname); errors.Is(err, os.ErrNotExist) {
 		log.Println("u-boot files copying skipped: ", err)
 	} else {
 		if err != nil {
@@ -43,9 +43,15 @@ func (b *BootDeploy) Run() error {
 		}
 	}
 
-	kernels, err := getKernelPath(b.outDir, b.devinfo.GenerateSystemdBoot == "true")
-	if err != nil {
-		return err
+	return bootDeploy(b.inDir, b.outDir)
+}
+
+func bootDeploy(workDir string, outDir string) error {
+	// boot-deploy expects the kernel to be in the same dir as initramfs.
+	// Assume that the kernel is in the output dir...
+	kernels, _ := filepath.Glob(filepath.Join(outDir, "vmlinuz*"))
+	if len(kernels) == 0 {
+		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
 	}
 
 	// Pick a kernel that does not have suffixes added by boot-deploy
@@ -65,7 +71,7 @@ func (b *BootDeploy) Run() error {
 	defer kernFd.Close()
 
 	kernFilename := path.Base(kernFile)
-	kernFileCopy, err := os.Create(filepath.Join(b.inDir, kernFilename))
+	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -81,8 +87,8 @@ func (b *BootDeploy) Run() error {
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
 		"-k", kernFilename,
-		"-d", b.inDir,
-		"-o", b.outDir,
+		"-d", workDir,
+		"-o", outDir,
 		"initramfs-extra")
 
 	cmd.Stdout = os.Stdout
@@ -94,25 +100,6 @@ func (b *BootDeploy) Run() error {
 	return nil
 }
 
-func getKernelPath(outDir string, zboot bool) ([]string, error) {
-	var kernels []string
-	if zboot {
-		kernels, _ = filepath.Glob(filepath.Join(outDir, "linux.efi"))
-		if len(kernels) > 0 {
-			return kernels, nil
-		}
-		// else fallback to vmlinuz* below
-	}
-
-	kernFile := "vmlinuz*"
-	kernels, _ = filepath.Glob(filepath.Join(outDir, kernFile))
-	if len(kernels) == 0 {
-		return nil, errors.New("Unable to find any kernels at " + filepath.Join(outDir, kernFile))
-	}
-
-	return kernels, nil
-}
-
 // Copy copies the file at srcFile path to a new file at dstFile path
 func copy(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)

internal/filelist/filelist.go

@@ -1,6 +1,12 @@
 package filelist
 
-import "sync"
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
+)
 
 type FileLister interface {
 	List() (*FileList, error)
@@ -45,6 +51,26 @@ func (f *FileList) Import(src *FileList) {
 	}
 }
 
+func (f *FileList) AddGlobbed(src string, dest string) error {
+	fFiles, err := misc.GetFiles([]string{src}, true)
+	if err != nil {
+		return fmt.Errorf("unable to add %q: %w", src, err)
+	}
+	// loop over all returned files from GetFile
+	for _, file := range fFiles {
+		if len(fFiles) > 1 {
+			// Glob with arbitrary subdirectories, so we need to
+			// remove the src path and prepend the dest path
+			f.Add(file, filepath.Join(dest, file[len(src):]))
+		} else {
+			// dest path specified, and only 1 file
+			f.Add(file, dest)
+		}
+	}
+
+	return nil
+}
+
 // iterate through the list and and send each one as a new File over the
 // returned channel
 func (f *FileList) IterItems() <-chan File {

internal/filelist/hookdirs/hookdirs.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 )
@@ -45,10 +44,6 @@ func (h *HookDirs) List() (*filelist.FileList, error) {
 		s := bufio.NewScanner(f)
 		for s.Scan() {
 			dir := s.Text()
-			if len(dir) == 0 || strings.HasPrefix(dir, "#") {
-				continue
-			}
-
 			files.Add(dir, dir)
 		}
 	}

internal/filelist/hookfiles/hookfiles.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type HookFiles struct {
@@ -58,29 +57,15 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		line := s.Text()
-		if len(line) == 0 || strings.HasPrefix(line, "#") {
-			continue
+		src, dest, has_dest := strings.Cut(s.Text(), ":")
+
+		if !has_dest {
+			dest = src
 		}
 
-		src, dest, has_dest := strings.Cut(line, ":")
-
-		fFiles, err := misc.GetFiles([]string{src}, true)
+		err := files.AddGlobbed(src, dest)
 		if err != nil {
-			return nil, fmt.Errorf("unable to add %q: %w", src, err)
-		}
-		// loop over all returned files from GetFile
-		for _, file := range fFiles {
-			if !has_dest {
-				files.Add(file, file)
-			} else if len(fFiles) > 1 {
-				// Don't support specifying dest if src was a glob
-				// NOTE: this could support this later...
-				files.Add(file, file)
-			} else {
-				// dest path specified, and only 1 file
-				files.Add(file, dest)
-			}
+			return nil, err
 		}
 	}
 

internal/filelist/hookscripts/hookscripts.go

@@ -35,8 +35,13 @@ func (h *HookScripts) List() (*filelist.FileList, error) {
 	}
 	for _, file := range fileInfo {
 		path := filepath.Join(h.scriptsDir, file.Name())
-		log.Printf("-- Including script: %s\n", path)
-		files.Add(path, filepath.Join(h.destPath, file.Name()))
+		if file.IsDir() {
+			log.Printf("-- Including dir %s\n", path)
+			files.AddGlobbed(filepath.Join(path, "*"), filepath.Join(h.destPath, file.Name()))
+		} else {
+			log.Printf("-- Including script: %s\n", path)
+			files.Add(path, filepath.Join(h.destPath, file.Name()))
+		}
 	}
 	return files, nil
 }

internal/filelist/initramfs/initramfs.go

@@ -9,7 +9,6 @@ import (
 // combining the output from them.
 type Initramfs struct {
 	features []filelist.FileLister
-	files    *filelist.FileList
 }
 
 // New returns a new Initramfs that generate a list of files based on the given
@@ -21,18 +20,15 @@ func New(features []filelist.FileLister) *Initramfs {
 }
 
 func (i *Initramfs) List() (*filelist.FileList, error) {
-	if i.files != nil {
-		return i.files, nil
-	}
-	i.files = filelist.NewFileList()
+	files := filelist.NewFileList()
 
 	for _, f := range i.features {
 		list, err := f.List()
 		if err != nil {
 			return nil, err
 		}
-		i.files.Import(list)
+		files.Import(list)
 	}
 
-	return i.files, nil
+	return files, nil
 }

internal/filelist/modules/modules.go

@@ -17,11 +17,14 @@ import (
 
 type Modules struct {
 	modulesListPath string
+	modulesList     []string
 }
 
-// New returns a new Modules that will read in lists of kernel modules in the given path.
-func New(modulesListPath string) *Modules {
+// New returns a new Modules that will use the given moduleto provide a list
+// of script files.
+func New(modulesList []string, modulesListPath string) *Modules {
 	return &Modules{
+		modulesList:     modulesList,
 		modulesListPath: modulesListPath,
 	}
 }
@@ -49,6 +52,20 @@ func (m *Modules) List() (*filelist.FileList, error) {
 		files.Add(file, file)
 	}
 
+	// slurp up given list of modules
+	if len(m.modulesList) > 0 {
+		log.Printf("-- Including kernel modules from deviceinfo")
+		for _, module := range m.modulesList {
+			if modFilelist, err := getModule(module, modDir); err != nil {
+				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
+			} else {
+				for _, file := range modFilelist {
+					files.Add(file, file)
+				}
+			}
+		}
+	}
+
 	// slurp up modules from lists in modulesListPath
 	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
@@ -78,9 +95,6 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
 		line := s.Text()
-		if len(line) == 0 || strings.HasPrefix(line, "#") {
-			continue
-		}
 		dir, file := filepath.Split(line)
 		if file == "" {
 			// item is a directory
@@ -118,9 +132,7 @@ func getModulesInDir(modPath string) (files []string, err error) {
 			// Unable to walk path
 			return err
 		}
-		// this assumes module names are in the format <name>.ko[.format],
-		// where ".format" (e.g. ".gz") is optional.
-		if !strings.Contains(".ko", path) {
+		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
 			return nil
 		}
 		files = append(files, path)
@@ -182,12 +194,7 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {
-		line := s.Text()
-		if len(line) == 0 || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		fields := strings.Fields(line)
+		fields := strings.Fields(s.Text())
 		if len(fields) == 0 {
 			continue
 		}

internal/filelist/osksdl/osksdl.go

@@ -0,0 +1,158 @@
+package osksdl
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
+)
+
+type OskSdl struct {
+	mesaDriver string
+}
+
+// New returns a new HookScripts that will use the given path to provide a list
+// of script files.
+func New(mesaDriverName string) *OskSdl {
+	return &OskSdl{
+		mesaDriver: mesaDriverName,
+	}
+}
+
+// Get a list of files and their dependencies related to supporting rootfs full
+// disk (d)encryption
+func (s *OskSdl) List() (*filelist.FileList, error) {
+	files := filelist.NewFileList()
+
+	if exists, err := misc.Exists("/usr/bin/osk-sdl"); !exists {
+		return files, nil
+	} else if err != nil {
+		return files, fmt.Errorf("received unexpected error when getting status for %q: %w", "/usr/bin/osk-sdl", err)
+	}
+
+	log.Println("- Including osk-sdl support")
+
+	confFiles := []string{
+		"/etc/osk.conf",
+		"/etc/ts.conf",
+		"/etc/pointercal",
+		"/etc/fb.modes",
+		"/etc/directfbrc",
+	}
+	confFileList, err := misc.GetFiles(confFiles, false)
+	if err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	}
+	for _, file := range confFileList {
+		files.Add(file, file)
+	}
+
+	// osk-sdl
+	oskFiles := []string{
+		"/usr/bin/osk-sdl",
+		"/sbin/cryptsetup",
+		"/usr/lib/libGL.so.1",
+	}
+	if oskFileList, err := misc.GetFiles(oskFiles, true); err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	} else {
+		for _, file := range oskFileList {
+			files.Add(file, file)
+		}
+	}
+
+	fontFile, err := getOskConfFontPath("/etc/osk.conf")
+	if err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
+	}
+	files.Add(fontFile, fontFile)
+
+	// Directfb
+	dfbFiles := []string{}
+	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
+		if filepath.Ext(path) == ".so" {
+			dfbFiles = append(dfbFiles, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
+	}
+	if dfbFileList, err := misc.GetFiles(dfbFiles, true); err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	} else {
+		for _, file := range dfbFileList {
+			files.Add(file, file)
+		}
+	}
+
+	// tslib
+	tslibFiles := []string{}
+	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
+		if filepath.Ext(path) == ".so" {
+			tslibFiles = append(tslibFiles, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
+	}
+	libts, _ := filepath.Glob("/usr/lib/libts*")
+	tslibFiles = append(tslibFiles, libts...)
+	if tslibFileList, err := misc.GetFiles(tslibFiles, true); err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	} else {
+		for _, file := range tslibFileList {
+			files.Add(file, file)
+		}
+	}
+
+	// mesa hw accel
+	if s.mesaDriver != "" {
+		mesaFiles := []string{
+			"/usr/lib/libEGL.so.1",
+			"/usr/lib/libGLESv2.so.2",
+			"/usr/lib/libgbm.so.1",
+			"/usr/lib/libudev.so.1",
+			"/usr/lib/xorg/modules/dri/" + s.mesaDriver + "_dri.so",
+		}
+		if mesaFileList, err := misc.GetFiles(mesaFiles, true); err != nil {
+			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+		} else {
+			for _, file := range mesaFileList {
+				files.Add(file, file)
+			}
+		}
+	}
+
+	return files, nil
+}
+
+func getOskConfFontPath(oskConfPath string) (string, error) {
+	var path string
+	f, err := os.Open(oskConfPath)
+	if err != nil {
+		return path, err
+	}
+	defer f.Close()
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		fields := strings.Fields(s.Text())
+		// "key = val" is 3 fields
+		if len(fields) > 2 && fields[0] == "keyboard-font" {
+			path = fields[2]
+		}
+	}
+	if exists, err := misc.Exists(path); !exists {
+		return path, fmt.Errorf("unable to find font: %s", path)
+	} else if err != nil {
+		return path, fmt.Errorf("received unexpected error when getting status for %q: %w", path, err)
+	}
+
+	return path, nil
+}

internal/misc/getfiles.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
-	"log"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 )
@@ -24,53 +22,29 @@ func GetFiles(list []string, required bool) (files []string, err error) {
 	return
 }
 
-// This function doesn't handle globs, use getFile() instead.
-func getFileNormalized(file string, required bool) (files []string, err error) {
-	fileInfo, err := os.Stat(file)
-
-	// Trying some fallbacks...
+func getFile(file string, required bool) (files []string, err error) {
+	// Expand glob expression
+	expanded, err := filepath.Glob(file)
 	if err != nil {
-		type triedResult struct {
-			file string
-			err error
-		}
-
-		triedFiles := make([]triedResult, 0, 1)
-
-		// Temporary fallback until alpine/pmOS usr-merge happened
-		// If a path starts with /bin or /sbin, also try /usr equivalent before giving up
-		if strings.HasPrefix(file, "/bin/") || strings.HasPrefix(file, "/sbin/") {
-			fileUsr := filepath.Join("/usr", file)
-			_, err := os.Stat(fileUsr);
-			if err == nil {
-				log.Printf("getFile: failed to find %q, but found it in %q. Please adjust the path.", file, fileUsr)
-				return getFileNormalized(fileUsr, required)
+		return
+	}
+	if len(expanded) > 0 && expanded[0] != file {
+		for _, path := range expanded {
+			if globFiles, err := getFile(path, required); err != nil {
+				return files, err
 			} else {
-				triedFiles = append(triedFiles, triedResult{fileUsr, err})
+				files = append(files, globFiles...)
 			}
 		}
+		return RemoveDuplicates(files), nil
+	}
 
-		{
-			// Check if there is a Zstd-compressed version of the file
-			fileZstd := file + ".zst" // .zst is the extension used by linux-firmware
-			_, err := os.Stat(fileZstd);
-			if err == nil {
-				return getFileNormalized(fileZstd, required)
-			} else {
-				triedFiles = append(triedFiles, triedResult{fileZstd, err})
-			}
-		}
-		
-		// Failed to find anything
+	fileInfo, err := os.Stat(file)
+	if err != nil {
 		if required {
-			failStrings := make([]string, 0, 2)
-			for _, result := range triedFiles {
-				failStrings = append(failStrings, fmt.Sprintf("\n - also tried %q: %v", result.file, result.err))
-			}
-			return files, fmt.Errorf("getFile: failed to stat file %q: %v%q", file, err, strings.Join(failStrings, ""))
-		} else {
-			return files, nil
+			return files, fmt.Errorf("getFile: failed to stat file %q: %w", file, err)
 		}
+		return files, nil
 	}
 
 	if fileInfo.IsDir() {
@@ -109,26 +83,6 @@ func getFileNormalized(file string, required bool) (files []string, err error) {
 	return
 }
 
-func getFile(file string, required bool) (files []string, err error) {
-	// Expand glob expression
-	expanded, err := filepath.Glob(file)
-	if err != nil {
-		return
-	}
-	if len(expanded) > 0 && expanded[0] != file {
-		for _, path := range expanded {
-			if globFiles, err := getFile(path, required); err != nil {
-				return files, err
-			} else {
-				files = append(files, globFiles...)
-			}
-		}
-		return RemoveDuplicates(files), nil
-	}
-
-	return getFileNormalized(file, required)
-}
-
 func getDeps(file string, parents map[string]struct{}) (files []string, err error) {
 
 	if _, found := parents[file]; found {

pkgs/deviceinfo/deviceinfo.go

@@ -11,42 +11,34 @@ import (
 	"os"
 	"reflect"
 	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type DeviceInfo struct {
 	InitfsCompression      string
 	InitfsExtraCompression string
+	MesaDriver             string
+	ModulesInitfs          string
 	UbootBoardname         string
-	GenerateSystemdBoot    string
 }
 
-// Reads the relevant entries from "file" into DeviceInfo struct
-// Any already-set entries will be overwriten if they are present
-// in "file"
-func (d *DeviceInfo) ReadDeviceinfo(file string) error {
-	if exists, err := misc.Exists(file); !exists {
-		return fmt.Errorf("%q not found, required by mkinitfs", file)
-	} else if err != nil {
-		return fmt.Errorf("unexpected error getting status for %q: %s", file, err)
-	}
+func ReadDeviceinfo(file string) (DeviceInfo, error) {
+	var deviceinfo DeviceInfo
 
 	fd, err := os.Open(file)
 	if err != nil {
-		return err
+		return deviceinfo, err
 	}
 	defer fd.Close()
 
-	if err := d.unmarshal(fd); err != nil {
-		return err
+	if err := unmarshal(fd, &deviceinfo); err != nil {
+		return deviceinfo, err
 	}
 
-	return nil
+	return deviceinfo, nil
 }
 
 // Unmarshals a deviceinfo into a DeviceInfo struct
-func (d *DeviceInfo) unmarshal(r io.Reader) error {
+func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
 	s := bufio.NewScanner(r)
 	for s.Scan() {
 		line := s.Text()
@@ -82,7 +74,7 @@ func (d *DeviceInfo) unmarshal(r io.Reader) error {
 			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
 		}
 
-		field := reflect.ValueOf(d).Elem().FieldByName(fieldName)
+		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
 		if !field.IsValid() {
 			// an option that meets the deviceinfo "specification", but isn't
 			// one we care about in this module

pkgs/deviceinfo/deviceinfo_test.go

@@ -10,28 +10,6 @@ import (
 	"testing"
 )
 
-// Test ReadDeviceinfo and the logic of reading from multiple files
-func TestReadDeviceinfo(t *testing.T) {
-	compression_expected := "gz -9"
-
-	var devinfo DeviceInfo
-	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
-	if !strings.Contains(err.Error(), "required by mkinitfs") {
-		t.Errorf("received an unexpected err: %s", err)
-	}
-	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-first")
-	if err != nil {
-		t.Errorf("received an unexpected err: %s", err)
-	}
-	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-msm")
-	if err != nil {
-		t.Errorf("received an unexpected err: %s", err)
-	}
-	if devinfo.InitfsCompression != compression_expected {
-		t.Errorf("expected %q, got: %q", compression_expected, devinfo.InitfsCompression)
-	}
-}
-
 // Test conversion of name to DeviceInfo struct field format
 func TestNameToField(t *testing.T) {
 	tables := []struct {
@@ -40,9 +18,9 @@ func TestNameToField(t *testing.T) {
 	}{
 		{"deviceinfo_dtb", "Dtb"},
 		{"dtb", "Dtb"},
-		{"deviceinfo_initfs_compression", "InitfsCompression"},
+		{"deviceinfo_modules_initfs", "ModulesInitfs"},
 		{"modules_initfs", "ModulesInitfs"},
-		{"deviceinfo_initfs_compression___", "InitfsCompression"},
+		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
 		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
@@ -63,12 +41,14 @@ func TestUnmarshal(t *testing.T) {
 		in       string
 		expected string
 	}{
-		{"InitfsCompression", "deviceinfo_initfs_compression=\"gzip:-9\"\n", "gzip:-9"},
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
 		// line with multiple '='
 		{"InitfsCompression", "deviceinfo_initfs_compression=zstd:--foo=1 -T0 --bar=bazz", "zstd:--foo=1 -T0 --bar=bazz"},
 		// empty option
-		{"InitfsCompression", "deviceinfo_initfs_compression=\"\"\n", ""},
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
 		// line with comment at the end
+		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
 		{"", "# this is a comment!\n", ""},
 		// empty lines are fine
 		{"", "", ""},
@@ -78,7 +58,7 @@ func TestUnmarshal(t *testing.T) {
 	var d DeviceInfo
 	for _, table := range tables {
 		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
-		if err := d.unmarshal(strings.NewReader(table.in)); err != nil {
+		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
 			t.Errorf("%s received an unexpected err: ", err)
 		}
 

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -1,2 +0,0 @@
-deviceinfo_initfs_compression="gz -9"
-deviceinfo_mesa_driver="panfrost"

pkgs/deviceinfo/test_resources/deviceinfo-msm

@@ -1 +0,0 @@
-deviceinfo_mesa_driver="msm"