misc: Also check for .zst-compressed variants of files (MR 49)

Initially I thought of breaking off the Stat + error check call into its
own function as to reduce repetition, but given that it's only useful in
this situation where it only happens twice anyway, I'm not sure it
actually would reduce complexity.

Additionally, this means that .zst-compressed variants of files will be
searched for in all contexts where this function is used. I'm not sure
this is desirable.

Tested and works with arrow-db820c. I didn't test it on actual hardware,
but I verified that the firmware ended up in the initramfs via
$ pmbootstrap initfs ls. I choose this device because it uses firmware
from linux-firmware and also needs said firmware present in the
initramfs.

Closes https://gitlab.com/postmarketOS/postmarketos-mkinitfs/-/issues/39

cmd/mkinitfs/main.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/archive"
@@ -20,7 +19,6 @@ import (
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/hookscripts"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/initramfs"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
@@ -49,18 +47,13 @@ func main() {
 
 	log.Default().SetFlags(log.Lmicroseconds)
 
-	deviceinfoFile := "/etc/deviceinfo"
-	if exists, err := misc.Exists(deviceinfoFile); !exists {
-		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
-		return
-	} else if err != nil {
-		retCode = 1
-		log.Printf("received unexpected error when getting status for %q: %s", deviceinfoFile, err)
-	}
-
-	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
-	if err != nil {
-		log.Println(err)
+	var devinfo deviceinfo.DeviceInfo
+	deverr_usr := devinfo.ReadDeviceinfo("/usr/share/deviceinfo/deviceinfo")
+	deverr_etc := devinfo.ReadDeviceinfo("/etc/deviceinfo")
+	if deverr_etc != nil && deverr_usr != nil {
+		log.Println("Error reading deviceinfo")
+		log.Println("\t/usr/share/deviceinfo/deviceinfo:", deverr_usr)
+		log.Println("\t/etc/deviceinfo:", deverr_etc)
 		retCode = 1
 		return
 	}
@@ -85,52 +78,83 @@ func main() {
 	defer func() {
 		e := os.RemoveAll(workDir)
 		if e != nil && err == nil {
-			err = e
-			retCode = 1
+			log.Println(e)
+			log.Println("unable to remove temporary work directory")
 		}
 	}()
 
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
+	//
+	// initramfs
+	//
 	// deviceinfo.InitfsCompression needs a little more post-processing
 	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	log.Printf("== Generating %s ==\n", "initramfs")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start := time.Now()
+	initramfsAr := archive.New(compressionFormat, compressionLevel)
+	initfs := initramfs.New([]filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
 		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
 		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
-		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
-		modules.New([]string{}, "/etc/mkinitfs/modules"),
-	}); err != nil {
+		modules.New("/usr/share/mkinitfs/modules"),
+		modules.New("/etc/mkinitfs/modules"),
+	})
+	if err := initramfsAr.AddItems(initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
+	if err := initramfsAr.Write(filepath.Join(workDir, "initramfs"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs")
 
+	//
+	// initramfs-extra
+	//
 	// deviceinfo.InitfsExtraCompression needs a little more post-processing
 	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
-	if err := generateArchive("initramfs-extra", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	log.Printf("== Generating %s ==\n", "initramfs-extra")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start = time.Now()
+	initramfsExtraAr := archive.New(compressionFormat, compressionLevel)
+	initfsExtra := initramfs.New([]filelist.FileLister{
 		hookfiles.New("/usr/share/mkinitfs/files-extra"),
 		hookfiles.New("/etc/mkinitfs/files-extra"),
 		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
 		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
-		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
-		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
-		osksdl.New(devinfo.MesaDriver),
-	}); err != nil {
+		modules.New("/usr/share/mkinitfs/modules-extra"),
+		modules.New("/etc/mkinitfs/modules-extra"),
+	})
+	if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs-extra")
 		retCode = 1
 		return
 	}
+	if err := initramfsExtraAr.Write(filepath.Join(workDir, "initramfs-extra"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs-extra")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs-extra")
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
-		if err := bootDeploy(workDir, *outDir, devinfo.UbootBoardname); err != nil {
+		if err := bootDeploy(workDir, *outDir, devinfo); err != nil {
 			log.Println(err)
 			log.Println("boot-deploy failed")
 			retCode = 1
@@ -139,30 +163,10 @@ func main() {
 	}
 }
 
-func bootDeploy(workDir, outDir, ubootBoardname string) error {
+func bootDeploy(workDir string, outDir string, devinfo deviceinfo.DeviceInfo) error {
 	log.Print("== Using boot-deploy to finalize/install files ==")
 	defer misc.TimeFunc(time.Now(), "boot-deploy")
 
-	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
+	bd := bootdeploy.New(workDir, outDir, devinfo)
 	return bd.Run()
 }
-
-func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
-	log.Printf("== Generating %s ==\n", name)
-	log.Printf("- Using compression format %s with level %q\n", format, level)
-
-	defer misc.TimeFunc(time.Now(), name)
-	a := archive.New(format, level)
-
-	fs := initramfs.New(features)
-	if err := a.AddItems(fs); err != nil {
-		return err
-	}
-
-	log.Println("- Writing and verifying archive: ", name)
-	if err := a.Write(filepath.Join(path, name), os.FileMode(0644)); err != nil {
-		return err
-	}
-
-	return nil
-}

doc/mkinitfs.1.scd

@@ -38,13 +38,13 @@ Design goals of this project are:
 The canonical deviceinfo "specification" is at
 https://wiki.postmarketos.org/wiki/Deviceinfo_reference
 
-mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
+mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
+*/etc/deviceinfo*, in that order. The following variables
 are *required* by mkinitfs:
 
+	- deviceinfo_generate_systemd_boot
 	- deviceinfo_initfs_compression
 	- deviceinfo_initfs_extra_compression
-	- deviceinfo_mesa_driver
-	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
 
 It is a design goal to keep the number of required variables from deviceinfo to
@@ -63,7 +63,7 @@ it are for constructing the initramfs archive.
 
 Configuration under */usr/share/mkinitfs* is intended to be managed by
 distributions, while configuration under */etc/mkinitfs* is for users to
-create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*. 
+create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*.
 
 ## /usr/share/mkinitfs/files, /etc/mkinitfs/files
 ## /usr/share/mkinitfs/files-extra, /etc/mkinitfs/files-extra
@@ -99,6 +99,9 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	path(s) under the relevant directory in */etc/mkinitfs*, and changing
 	the destination path.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/hooks, /etc/mkinitfs/hooks
 ## /usr/share/mkinitfs/hooks-extra*, /etc/mkinitfs/hooks-extra
 
@@ -120,12 +123,18 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	Modules are installed in the initramfs archive under the same path they
 	exist on the system where mkinitfs is executed.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/dirs, /etc/mkinitfs/dirs
 
 	Files with the *.dirs* extension in these directories are lists of
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 # BOOT-DEPLOY
 
 After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
@@ -136,7 +145,7 @@ search for the app. The following commandline options are passed to it:
 		Currently this is hardcoded to be "initramfs"
 
 	*-k* <kernel filename>
-	
+
 	*-d* <work directory>
 
 		Path to the directory containing the build artifacts from mkinitfs.

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.15.12
+	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/ulikunitz/xz v0.5.10
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 )

go.sum

@@ -2,6 +2,8 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RS
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
 github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
 github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=

internal/archive/archive.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/cavaliercoder/go-cpio"
 	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/ulikunitz/xz"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
@@ -28,6 +29,7 @@ type CompressFormat string
 const (
 	FormatGzip CompressFormat = "gzip"
 	FormatLzma CompressFormat = "lzma"
+	FormatLz4  CompressFormat = "lz4"
 	FormatZstd CompressFormat = "zstd"
 	FormatNone CompressFormat = "none"
 )
@@ -104,6 +106,7 @@ func ExtractFormatLevel(s string) (format CompressFormat, level CompressLevel) {
 	case FormatLzma:
 		log.Println("Format lzma doesn't support a compression level, using default settings")
 		level = LevelDefault
+	case FormatLz4:
 	case FormatNone:
 	case FormatZstd:
 	default:
@@ -200,6 +203,38 @@ func (archive *Archive) AddItems(flister filelist.FileLister) error {
 	return nil
 }
 
+// AddItemsExclude is like AddItems, but takes a second FileLister that lists
+// items that should not be added to the archive from the first FileLister
+func (archive *Archive) AddItemsExclude(flister filelist.FileLister, exclude filelist.FileLister) error {
+	list, err := flister.List()
+	if err != nil {
+		return err
+	}
+
+	excludeList, err := exclude.List()
+	if err != nil {
+		return err
+	}
+
+	for i := range list.IterItems() {
+		dest, found := excludeList.Get(i.Source)
+
+		if found {
+			if i.Dest != dest {
+				found = false
+			}
+		}
+
+		if !found {
+			if err := archive.AddItem(i.Source, i.Dest); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // Adds the given file or directory at "source" to the archive at "dest"
 func (archive *Archive) AddItem(source string, dest string) error {
 
@@ -316,6 +351,23 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 		if err != nil {
 			return err
 		}
+	case FormatLz4:
+		// The default compression for the lz4 library is Fast, and
+		// they don't define a Default level otherwise
+		level := lz4.Fast
+		switch archive.compress_level {
+		case LevelBest:
+			level = lz4.Level9
+		case LevelFast:
+			level = lz4.Fast
+		}
+
+		var writer = lz4.NewWriter(fd)
+		err = writer.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(level))
+		if err != nil {
+			return err
+		}
+		compressor = writer
 	case FormatNone:
 		compressor = fd
 	case FormatZstd:

internal/archive/archive_test.go

@@ -249,6 +249,12 @@ func TestExtractFormatLevel(t *testing.T) {
 			expectedFormat: FormatLzma,
 			expectedLevel:  LevelDefault,
 		},
+		{
+			name:           "lz4, fast",
+			in:             "lz4:fast",
+			expectedFormat: FormatLz4,
+			expectedLevel:  LevelFast,
+		},
 		{
 			name:           "none",
 			in:             "none",

internal/bootdeploy/bootdeploy.go

@@ -10,32 +10,32 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
 
 type BootDeploy struct {
-	inDir          string
-	outDir         string
-	ubootBoardname string
+	inDir   string
+	outDir  string
+	devinfo deviceinfo.DeviceInfo
 }
 
 // New returns a new BootDeploy, which then runs:
 //
 //	boot-deploy -d indir -o outDir
 //
-// ubootBoardname is used for copying in some u-boot files prior to running
-// boot-deploy. This is optional, passing an empty string is ok if this is not
-// needed.
-func New(inDir, outDir, ubootBoardname string) *BootDeploy {
+// devinfo is used to access some deviceinfo values, such as UbootBoardname
+// and GenerateSystemdBoot
+func New(inDir string, outDir string, devinfo deviceinfo.DeviceInfo) *BootDeploy {
 	return &BootDeploy{
-		inDir:          inDir,
-		outDir:         outDir,
-		ubootBoardname: ubootBoardname,
+		inDir:   inDir,
+		outDir:  outDir,
+		devinfo: devinfo,
 	}
 }
 
 func (b *BootDeploy) Run() error {
-
-	if err := copyUbootFiles(b.inDir, b.ubootBoardname); errors.Is(err, os.ErrNotExist) {
+	if err := copyUbootFiles(b.inDir, b.devinfo.UbootBoardname); errors.Is(err, os.ErrNotExist) {
 		log.Println("u-boot files copying skipped: ", err)
 	} else {
 		if err != nil {
@@ -43,15 +43,9 @@ func (b *BootDeploy) Run() error {
 		}
 	}
 
-	return bootDeploy(b.inDir, b.outDir)
-}
-
-func bootDeploy(workDir string, outDir string) error {
-	// boot-deploy expects the kernel to be in the same dir as initramfs.
-	// Assume that the kernel is in the output dir...
-	kernels, _ := filepath.Glob(filepath.Join(outDir, "vmlinuz*"))
-	if len(kernels) == 0 {
-		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
+	kernels, err := getKernelPath(b.outDir, b.devinfo.GenerateSystemdBoot == "true")
+	if err != nil {
+		return err
 	}
 
 	// Pick a kernel that does not have suffixes added by boot-deploy
@@ -71,7 +65,7 @@ func bootDeploy(workDir string, outDir string) error {
 	defer kernFd.Close()
 
 	kernFilename := path.Base(kernFile)
-	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
+	kernFileCopy, err := os.Create(filepath.Join(b.inDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -87,8 +81,8 @@ func bootDeploy(workDir string, outDir string) error {
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
 		"-k", kernFilename,
-		"-d", workDir,
-		"-o", outDir,
+		"-d", b.inDir,
+		"-o", b.outDir,
 		"initramfs-extra")
 
 	cmd.Stdout = os.Stdout
@@ -100,6 +94,25 @@ func bootDeploy(workDir string, outDir string) error {
 	return nil
 }
 
+func getKernelPath(outDir string, zboot bool) ([]string, error) {
+	var kernels []string
+	if zboot {
+		kernels, _ = filepath.Glob(filepath.Join(outDir, "linux.efi"))
+		if len(kernels) > 0 {
+			return kernels, nil
+		}
+		// else fallback to vmlinuz* below
+	}
+
+	kernFile := "vmlinuz*"
+	kernels, _ = filepath.Glob(filepath.Join(outDir, kernFile))
+	if len(kernels) == 0 {
+		return nil, errors.New("Unable to find any kernels at " + filepath.Join(outDir, kernFile))
+	}
+
+	return kernels, nil
+}
+
 // Copy copies the file at srcFile path to a new file at dstFile path
 func copy(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)

internal/filelist/filelist.go

@@ -1,12 +1,6 @@
 package filelist
 
-import (
-	"fmt"
-	"path/filepath"
-	"sync"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-)
+import "sync"
 
 type FileLister interface {
 	List() (*FileList, error)
@@ -51,26 +45,6 @@ func (f *FileList) Import(src *FileList) {
 	}
 }
 
-func (f *FileList) AddGlobbed(src string, dest string) error {
-	fFiles, err := misc.GetFiles([]string{src}, true)
-	if err != nil {
-		return fmt.Errorf("unable to add %q: %w", src, err)
-	}
-	// loop over all returned files from GetFile
-	for _, file := range fFiles {
-		if len(fFiles) > 1 {
-			// Glob with arbitrary subdirectories, so we need to
-			// remove the src path and prepend the dest path
-			f.Add(file, filepath.Join(dest, file[len(src):]))
-		} else {
-			// dest path specified, and only 1 file
-			f.Add(file, dest)
-		}
-	}
-
-	return nil
-}
-
 // iterate through the list and and send each one as a new File over the
 // returned channel
 func (f *FileList) IterItems() <-chan File {

internal/filelist/hookdirs/hookdirs.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 )
@@ -44,6 +45,10 @@ func (h *HookDirs) List() (*filelist.FileList, error) {
 		s := bufio.NewScanner(f)
 		for s.Scan() {
 			dir := s.Text()
+			if len(dir) == 0 || strings.HasPrefix(dir, "#") {
+				continue
+			}
+
 			files.Add(dir, dir)
 		}
 	}

internal/filelist/hookfiles/hookfiles.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type HookFiles struct {
@@ -57,15 +58,29 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		src, dest, has_dest := strings.Cut(s.Text(), ":")
-
-		if !has_dest {
-			dest = src
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
 		}
 
-		err := files.AddGlobbed(src, dest)
+		src, dest, has_dest := strings.Cut(line, ":")
+
+		fFiles, err := misc.GetFiles([]string{src}, true)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("unable to add %q: %w", src, err)
+		}
+		// loop over all returned files from GetFile
+		for _, file := range fFiles {
+			if !has_dest {
+				files.Add(file, file)
+			} else if len(fFiles) > 1 {
+				// Don't support specifying dest if src was a glob
+				// NOTE: this could support this later...
+				files.Add(file, file)
+			} else {
+				// dest path specified, and only 1 file
+				files.Add(file, dest)
+			}
 		}
 	}
 

internal/filelist/hookscripts/hookscripts.go

@@ -35,13 +35,8 @@ func (h *HookScripts) List() (*filelist.FileList, error) {
 	}
 	for _, file := range fileInfo {
 		path := filepath.Join(h.scriptsDir, file.Name())
-		if file.IsDir() {
-			log.Printf("-- Including dir %s\n", path)
-			files.AddGlobbed(filepath.Join(path, "*"), filepath.Join(h.destPath, file.Name()))
-		} else {
-			log.Printf("-- Including script: %s\n", path)
-			files.Add(path, filepath.Join(h.destPath, file.Name()))
-		}
+		log.Printf("-- Including script: %s\n", path)
+		files.Add(path, filepath.Join(h.destPath, file.Name()))
 	}
 	return files, nil
 }

internal/filelist/initramfs/initramfs.go

@@ -9,6 +9,7 @@ import (
 // combining the output from them.
 type Initramfs struct {
 	features []filelist.FileLister
+	files    *filelist.FileList
 }
 
 // New returns a new Initramfs that generate a list of files based on the given
@@ -20,15 +21,18 @@ func New(features []filelist.FileLister) *Initramfs {
 }
 
 func (i *Initramfs) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
+	if i.files != nil {
+		return i.files, nil
+	}
+	i.files = filelist.NewFileList()
 
 	for _, f := range i.features {
 		list, err := f.List()
 		if err != nil {
 			return nil, err
 		}
-		files.Import(list)
+		i.files.Import(list)
 	}
 
-	return files, nil
+	return i.files, nil
 }

internal/filelist/modules/modules.go

@@ -17,14 +17,11 @@ import (
 
 type Modules struct {
 	modulesListPath string
-	modulesList     []string
 }
 
-// New returns a new Modules that will use the given moduleto provide a list
-// of script files.
-func New(modulesList []string, modulesListPath string) *Modules {
+// New returns a new Modules that will read in lists of kernel modules in the given path.
+func New(modulesListPath string) *Modules {
 	return &Modules{
-		modulesList:     modulesList,
 		modulesListPath: modulesListPath,
 	}
 }
@@ -52,20 +49,6 @@ func (m *Modules) List() (*filelist.FileList, error) {
 		files.Add(file, file)
 	}
 
-	// slurp up given list of modules
-	if len(m.modulesList) > 0 {
-		log.Printf("-- Including kernel modules from deviceinfo")
-		for _, module := range m.modulesList {
-			if modFilelist, err := getModule(module, modDir); err != nil {
-				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
-			} else {
-				for _, file := range modFilelist {
-					files.Add(file, file)
-				}
-			}
-		}
-	}
-
 	// slurp up modules from lists in modulesListPath
 	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
@@ -95,6 +78,9 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
 		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
 		dir, file := filepath.Split(line)
 		if file == "" {
 			// item is a directory
@@ -132,7 +118,9 @@ func getModulesInDir(modPath string) (files []string, err error) {
 			// Unable to walk path
 			return err
 		}
-		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
+		// this assumes module names are in the format <name>.ko[.format],
+		// where ".format" (e.g. ".gz") is optional.
+		if !strings.Contains(".ko", path) {
 			return nil
 		}
 		files = append(files, path)
@@ -194,7 +182,12 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {
-		fields := strings.Fields(s.Text())
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		fields := strings.Fields(line)
 		if len(fields) == 0 {
 			continue
 		}

internal/filelist/osksdl/osksdl.go

@@ -1,158 +0,0 @@
-package osksdl
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-)
-
-type OskSdl struct {
-	mesaDriver string
-}
-
-// New returns a new HookScripts that will use the given path to provide a list
-// of script files.
-func New(mesaDriverName string) *OskSdl {
-	return &OskSdl{
-		mesaDriver: mesaDriverName,
-	}
-}
-
-// Get a list of files and their dependencies related to supporting rootfs full
-// disk (d)encryption
-func (s *OskSdl) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
-
-	if exists, err := misc.Exists("/usr/bin/osk-sdl"); !exists {
-		return files, nil
-	} else if err != nil {
-		return files, fmt.Errorf("received unexpected error when getting status for %q: %w", "/usr/bin/osk-sdl", err)
-	}
-
-	log.Println("- Including osk-sdl support")
-
-	confFiles := []string{
-		"/etc/osk.conf",
-		"/etc/ts.conf",
-		"/etc/pointercal",
-		"/etc/fb.modes",
-		"/etc/directfbrc",
-	}
-	confFileList, err := misc.GetFiles(confFiles, false)
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	}
-	for _, file := range confFileList {
-		files.Add(file, file)
-	}
-
-	// osk-sdl
-	oskFiles := []string{
-		"/usr/bin/osk-sdl",
-		"/sbin/cryptsetup",
-		"/usr/lib/libGL.so.1",
-	}
-	if oskFileList, err := misc.GetFiles(oskFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range oskFileList {
-			files.Add(file, file)
-		}
-	}
-
-	fontFile, err := getOskConfFontPath("/etc/osk.conf")
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
-	}
-	files.Add(fontFile, fontFile)
-
-	// Directfb
-	dfbFiles := []string{}
-	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			dfbFiles = append(dfbFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
-	}
-	if dfbFileList, err := misc.GetFiles(dfbFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range dfbFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// tslib
-	tslibFiles := []string{}
-	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			tslibFiles = append(tslibFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
-	}
-	libts, _ := filepath.Glob("/usr/lib/libts*")
-	tslibFiles = append(tslibFiles, libts...)
-	if tslibFileList, err := misc.GetFiles(tslibFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range tslibFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// mesa hw accel
-	if s.mesaDriver != "" {
-		mesaFiles := []string{
-			"/usr/lib/libEGL.so.1",
-			"/usr/lib/libGLESv2.so.2",
-			"/usr/lib/libgbm.so.1",
-			"/usr/lib/libudev.so.1",
-			"/usr/lib/xorg/modules/dri/" + s.mesaDriver + "_dri.so",
-		}
-		if mesaFileList, err := misc.GetFiles(mesaFiles, true); err != nil {
-			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-		} else {
-			for _, file := range mesaFileList {
-				files.Add(file, file)
-			}
-		}
-	}
-
-	return files, nil
-}
-
-func getOskConfFontPath(oskConfPath string) (string, error) {
-	var path string
-	f, err := os.Open(oskConfPath)
-	if err != nil {
-		return path, err
-	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		fields := strings.Fields(s.Text())
-		// "key = val" is 3 fields
-		if len(fields) > 2 && fields[0] == "keyboard-font" {
-			path = fields[2]
-		}
-	}
-	if exists, err := misc.Exists(path); !exists {
-		return path, fmt.Errorf("unable to find font: %s", path)
-	} else if err != nil {
-		return path, fmt.Errorf("received unexpected error when getting status for %q: %w", path, err)
-	}
-
-	return path, nil
-}

internal/misc/getfiles.go

@@ -41,10 +41,22 @@ func getFile(file string, required bool) (files []string, err error) {
 
 	fileInfo, err := os.Stat(file)
 	if err != nil {
-		if required {
-			return files, fmt.Errorf("getFile: failed to stat file %q: %w", file, err)
+		// Check if there is a Zstd-compressed version of the file
+		fileZstd := file + ".zst" // .zst is the extension used by linux-firmware
+		fileInfoZstd, errZstd := os.Stat(fileZstd)
+
+		if errZstd == nil {
+			file = fileZstd
+			fileInfo = fileInfoZstd
+			// Unset nil so we don't retain the error from the os.Stat call for the uncompressed version.
+			err = nil
+		} else {
+			if required {
+				return files, fmt.Errorf("getFile: failed to stat file %q: %w (also tried %q: %w)", file, err, fileZstd, errZstd)
+			}
+
+			return files, nil
 		}
-		return files, nil
 	}
 
 	if fileInfo.IsDir() {

pkgs/deviceinfo/deviceinfo.go

@@ -11,34 +11,42 @@ import (
 	"os"
 	"reflect"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type DeviceInfo struct {
 	InitfsCompression      string
 	InitfsExtraCompression string
-	MesaDriver             string
-	ModulesInitfs          string
 	UbootBoardname         string
+	GenerateSystemdBoot    string
 }
 
-func ReadDeviceinfo(file string) (DeviceInfo, error) {
-	var deviceinfo DeviceInfo
+// Reads the relevant entries from "file" into DeviceInfo struct
+// Any already-set entries will be overwriten if they are present
+// in "file"
+func (d *DeviceInfo) ReadDeviceinfo(file string) error {
+	if exists, err := misc.Exists(file); !exists {
+		return fmt.Errorf("%q not found, required by mkinitfs", file)
+	} else if err != nil {
+		return fmt.Errorf("unexpected error getting status for %q: %s", file, err)
+	}
 
 	fd, err := os.Open(file)
 	if err != nil {
-		return deviceinfo, err
+		return err
 	}
 	defer fd.Close()
 
-	if err := unmarshal(fd, &deviceinfo); err != nil {
-		return deviceinfo, err
+	if err := d.unmarshal(fd); err != nil {
+		return err
 	}
 
-	return deviceinfo, nil
+	return nil
 }
 
 // Unmarshals a deviceinfo into a DeviceInfo struct
-func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
+func (d *DeviceInfo) unmarshal(r io.Reader) error {
 	s := bufio.NewScanner(r)
 	for s.Scan() {
 		line := s.Text()
@@ -74,7 +82,7 @@ func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
 			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
 		}
 
-		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
+		field := reflect.ValueOf(d).Elem().FieldByName(fieldName)
 		if !field.IsValid() {
 			// an option that meets the deviceinfo "specification", but isn't
 			// one we care about in this module

pkgs/deviceinfo/deviceinfo_test.go

@@ -10,6 +10,28 @@ import (
 	"testing"
 )
 
+// Test ReadDeviceinfo and the logic of reading from multiple files
+func TestReadDeviceinfo(t *testing.T) {
+	compression_expected := "gz -9"
+
+	var devinfo DeviceInfo
+	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
+	if !strings.Contains(err.Error(), "required by mkinitfs") {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-first")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-msm")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	if devinfo.InitfsCompression != compression_expected {
+		t.Errorf("expected %q, got: %q", compression_expected, devinfo.InitfsCompression)
+	}
+}
+
 // Test conversion of name to DeviceInfo struct field format
 func TestNameToField(t *testing.T) {
 	tables := []struct {
@@ -18,9 +40,9 @@ func TestNameToField(t *testing.T) {
 	}{
 		{"deviceinfo_dtb", "Dtb"},
 		{"dtb", "Dtb"},
-		{"deviceinfo_modules_initfs", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression", "InitfsCompression"},
 		{"modules_initfs", "ModulesInitfs"},
-		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression___", "InitfsCompression"},
 		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
@@ -41,14 +63,12 @@ func TestUnmarshal(t *testing.T) {
 		in       string
 		expected string
 	}{
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"gzip:-9\"\n", "gzip:-9"},
 		// line with multiple '='
 		{"InitfsCompression", "deviceinfo_initfs_compression=zstd:--foo=1 -T0 --bar=bazz", "zstd:--foo=1 -T0 --bar=bazz"},
 		// empty option
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"\"\n", ""},
 		// line with comment at the end
-		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
 		{"", "# this is a comment!\n", ""},
 		// empty lines are fine
 		{"", "", ""},
@@ -58,7 +78,7 @@ func TestUnmarshal(t *testing.T) {
 	var d DeviceInfo
 	for _, table := range tables {
 		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
-		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
+		if err := d.unmarshal(strings.NewReader(table.in)); err != nil {
 			t.Errorf("%s received an unexpected err: ", err)
 		}
 

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -0,0 +1,2 @@
+deviceinfo_initfs_compression="gz -9"
+deviceinfo_mesa_driver="panfrost"

pkgs/deviceinfo/test_resources/deviceinfo-msm

@@ -0,0 +1 @@
+deviceinfo_mesa_driver="msm"