misc: also check wheter binaries from /{s,}bin are /usr

This should hopefully avoid failing to build the initramfs
when important binaries like /bin/busybox get moved to /usr.

Also complain loudly when there's a path mismatch, so people notice
it.

cmd/mkinitfs/main.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/archive"
@@ -20,7 +19,6 @@ import (
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/hookscripts"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/initramfs"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
@@ -49,18 +47,13 @@ func main() {
 
 	log.Default().SetFlags(log.Lmicroseconds)
 
-	deviceinfoFile := "/etc/deviceinfo"
-	if exists, err := misc.Exists(deviceinfoFile); !exists {
-		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
-		return
-	} else if err != nil {
-		retCode = 1
-		log.Printf("received unexpected error when getting status for %q: %s", deviceinfoFile, err)
-	}
-
-	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
-	if err != nil {
-		log.Println(err)
+	var devinfo deviceinfo.DeviceInfo
+	deverr_usr := devinfo.ReadDeviceinfo("/usr/share/deviceinfo/deviceinfo")
+	deverr_etc := devinfo.ReadDeviceinfo("/etc/deviceinfo")
+	if deverr_etc != nil && deverr_usr != nil {
+		log.Println("Error reading deviceinfo")
+		log.Println("\t/usr/share/deviceinfo/deviceinfo:", deverr_usr)
+		log.Println("\t/etc/deviceinfo:", deverr_etc)
 		retCode = 1
 		return
 	}
@@ -85,52 +78,83 @@ func main() {
 	defer func() {
 		e := os.RemoveAll(workDir)
 		if e != nil && err == nil {
-			err = e
-			retCode = 1
+			log.Println(e)
+			log.Println("unable to remove temporary work directory")
 		}
 	}()
 
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
+	//
+	// initramfs
+	//
 	// deviceinfo.InitfsCompression needs a little more post-processing
 	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	log.Printf("== Generating %s ==\n", "initramfs")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start := time.Now()
+	initramfsAr := archive.New(compressionFormat, compressionLevel)
+	initfs := initramfs.New([]filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
 		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
 		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
-		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
-		modules.New([]string{}, "/etc/mkinitfs/modules"),
-	}); err != nil {
+		modules.New("/usr/share/mkinitfs/modules"),
+		modules.New("/etc/mkinitfs/modules"),
+	})
+	if err := initramfsAr.AddItems(initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
+	if err := initramfsAr.Write(filepath.Join(workDir, "initramfs"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs")
 
+	//
+	// initramfs-extra
+	//
 	// deviceinfo.InitfsExtraCompression needs a little more post-processing
 	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
-	if err := generateArchive("initramfs-extra", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	log.Printf("== Generating %s ==\n", "initramfs-extra")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start = time.Now()
+	initramfsExtraAr := archive.New(compressionFormat, compressionLevel)
+	initfsExtra := initramfs.New([]filelist.FileLister{
 		hookfiles.New("/usr/share/mkinitfs/files-extra"),
 		hookfiles.New("/etc/mkinitfs/files-extra"),
 		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
 		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
-		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
-		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
-		osksdl.New(devinfo.MesaDriver),
-	}); err != nil {
+		modules.New("/usr/share/mkinitfs/modules-extra"),
+		modules.New("/etc/mkinitfs/modules-extra"),
+	})
+	if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs-extra")
 		retCode = 1
 		return
 	}
+	if err := initramfsExtraAr.Write(filepath.Join(workDir, "initramfs-extra"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs-extra")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs-extra")
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
-		if err := bootDeploy(workDir, *outDir, devinfo.UbootBoardname); err != nil {
+		if err := bootDeploy(workDir, *outDir, devinfo); err != nil {
 			log.Println(err)
 			log.Println("boot-deploy failed")
 			retCode = 1
@@ -139,33 +163,10 @@ func main() {
 	}
 }
 
-func bootDeploy(workDir, outDir, ubootBoardname string) error {
+func bootDeploy(workDir string, outDir string, devinfo deviceinfo.DeviceInfo) error {
 	log.Print("== Using boot-deploy to finalize/install files ==")
 	defer misc.TimeFunc(time.Now(), "boot-deploy")
 
-	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
+	bd := bootdeploy.New(workDir, outDir, devinfo)
 	return bd.Run()
 }
-
-func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
-	log.Printf("== Generating %s ==\n", name)
-	log.Printf("- Using compression format %s with level %q\n", format, level)
-
-	defer misc.TimeFunc(time.Now(), name)
-	a, err := archive.New(format, level)
-	if err != nil {
-		return err
-	}
-
-	fs := initramfs.New(features)
-	if err := a.AddItems(fs); err != nil {
-		return err
-	}
-
-	log.Println("- Writing and verifying archive: ", name)
-	if err := a.Write(filepath.Join(path, name), os.FileMode(0644)); err != nil {
-		return err
-	}
-
-	return nil
-}

doc/mkinitfs.1.scd

@@ -24,18 +24,27 @@ is purely to generate the archive(s). mkinitfs does call *boot-deploy* after
 creating the archive(s), in order to install/deploy them and any other relevant
 boot-related items onto the system.
 
+Design goals of this project are:
+
+	- Support as many distros as possible
+	- Simplify configuration, while still giving multiple opportunities to set or override defaults
+	- Execute an external app to do any boot install/setup finalization
+		- One such app is here: https://gitlab.com/postmarketOS/boot-deploy
+		- But implementation can be anything, see the section on *BOOT-DEPLOY*
+		for more info
+
 # DEVICEINFO
 
 The canonical deviceinfo "specification" is at
 https://wiki.postmarketos.org/wiki/Deviceinfo_reference
 
-mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
+mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
+*/etc/deviceinfo*, in that order. The following variables
 are *required* by mkinitfs:
 
+	- deviceinfo_generate_systemd_boot
 	- deviceinfo_initfs_compression
 	- deviceinfo_initfs_extra_compression
-	- deviceinfo_mesa_driver
-	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
 
 It is a design goal to keep the number of required variables from deviceinfo to
@@ -45,6 +54,36 @@ a bare minimum, and to require only variables that don't hold lists of things.
 necessary tools to extract the configured archive format are in the initramfs
 archive.
 
+# ARCHIVE COMPRESSION
+
+Archive compression parameters are specified in the
+*deviceinfo_initfs_compression* and *deviceinfo_initfs_extra_compression*
+deviceinfo variables. Their values do not have to match, but special
+consideration should be taken since some formats may require additional kernel
+options or tools in the initramfs to support it.
+
+Supported compression *formats* for mkinitfs are:
+
+	- gzip
+	- lz4
+	- lzma
+	- none
+	- zstd
+
+Supported compression *levels* for mkinitfs:
+
+	- best
+	- default
+	- fast
+
+The value of these variables follows this syntax: *<format>:<level>*. For
+example, *zstd* with the *fast* compression level would be:
+*deviceinfo_initfs_compression="zstd:fast"*
+
+Defaults to *gzip* and *default* for both archives if format and/or level is
+unsupported or omitted.
+
+
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and
@@ -90,6 +129,9 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	path(s) under the relevant directory in */etc/mkinitfs*, and changing
 	the destination path.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/hooks, /etc/mkinitfs/hooks
 ## /usr/share/mkinitfs/hooks-extra*, /etc/mkinitfs/hooks-extra
 
@@ -111,12 +153,42 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	Modules are installed in the initramfs archive under the same path they
 	exist on the system where mkinitfs is executed.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/dirs, /etc/mkinitfs/dirs
 
 	Files with the *.dirs* extension in these directories are lists of
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
+# BOOT-DEPLOY
+
+After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
+search for the app. The following commandline options are passed to it:
+
+	*-i* <initramfs filename>
+
+		Currently this is hardcoded to be "initramfs"
+
+	*-k* <kernel filename>
+
+	*-d* <work directory>
+
+		Path to the directory containing the build artifacts from mkinitfs.
+
+	*-o* <destination directory>
+
+		Path to the directory that boot-deploy should use as its root when
+		installing files.
+
+	*initramfs-extra*
+
+		This string is the filename of the initramfs-extra archive.
+
 # AUTHORS
 
 *Clayton Craft* <clayton@craftyguy.net>

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.15.12
+	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/ulikunitz/xz v0.5.10
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 )

go.sum

@@ -2,6 +2,8 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RS
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
 github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
 github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=

internal/archive/archive.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/cavaliercoder/go-cpio"
 	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/ulikunitz/xz"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
@@ -28,6 +29,7 @@ type CompressFormat string
 const (
 	FormatGzip CompressFormat = "gzip"
 	FormatLzma CompressFormat = "lzma"
+	FormatLz4  CompressFormat = "lz4"
 	FormatZstd CompressFormat = "zstd"
 	FormatNone CompressFormat = "none"
 )
@@ -44,14 +46,14 @@ const (
 )
 
 type Archive struct {
-	items           archiveItems
 	cpioWriter      *cpio.Writer
 	buf             *bytes.Buffer
 	compress_format CompressFormat
 	compress_level  CompressLevel
+	items           archiveItems
 }
 
-func New(format CompressFormat, level CompressLevel) (*Archive, error) {
+func New(format CompressFormat, level CompressLevel) *Archive {
 	buf := new(bytes.Buffer)
 	archive := &Archive{
 		cpioWriter:      cpio.NewWriter(buf),
@@ -60,12 +62,12 @@ func New(format CompressFormat, level CompressLevel) (*Archive, error) {
 		compress_level:  level,
 	}
 
-	return archive, nil
+	return archive
 }
 
 type archiveItem struct {
-	sourcePath string
 	header     *cpio.Header
+	sourcePath string
 }
 
 type archiveItems struct {
@@ -104,6 +106,7 @@ func ExtractFormatLevel(s string) (format CompressFormat, level CompressLevel) {
 	case FormatLzma:
 		log.Println("Format lzma doesn't support a compression level, using default settings")
 		level = LevelDefault
+	case FormatLz4:
 	case FormatNone:
 	case FormatZstd:
 	default:
@@ -184,10 +187,11 @@ func (archive *Archive) Write(path string, mode os.FileMode) error {
 	return nil
 }
 
-// Adds the given items in the map to the archive. The map format is {source path:dest path}.
-// Internally this just calls AddItem on each key,value pair in the map.
-func (archive *Archive) AddItems(f filelist.FileLister) error {
-	list, err := f.List()
+// AddItems adds the given items in the map to the archive. The map format is
+// {source path:dest path}. Internally this just calls AddItem on each
+// key,value pair in the map.
+func (archive *Archive) AddItems(flister filelist.FileLister) error {
+	list, err := flister.List()
 	if err != nil {
 		return err
 	}
@@ -199,6 +203,38 @@ func (archive *Archive) AddItems(f filelist.FileLister) error {
 	return nil
 }
 
+// AddItemsExclude is like AddItems, but takes a second FileLister that lists
+// items that should not be added to the archive from the first FileLister
+func (archive *Archive) AddItemsExclude(flister filelist.FileLister, exclude filelist.FileLister) error {
+	list, err := flister.List()
+	if err != nil {
+		return err
+	}
+
+	excludeList, err := exclude.List()
+	if err != nil {
+		return err
+	}
+
+	for i := range list.IterItems() {
+		dest, found := excludeList.Get(i.Source)
+
+		if found {
+			if i.Dest != dest {
+				found = false
+			}
+		}
+
+		if !found {
+			if err := archive.AddItem(i.Source, i.Dest); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // Adds the given file or directory at "source" to the archive at "dest"
 func (archive *Archive) AddItem(source string, dest string) error {
 
@@ -315,6 +351,23 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 		if err != nil {
 			return err
 		}
+	case FormatLz4:
+		// The default compression for the lz4 library is Fast, and
+		// they don't define a Default level otherwise
+		level := lz4.Fast
+		switch archive.compress_level {
+		case LevelBest:
+			level = lz4.Level9
+		case LevelFast:
+			level = lz4.Fast
+		}
+
+		var writer = lz4.NewWriter(fd)
+		err = writer.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(level))
+		if err != nil {
+			return err
+		}
+		compressor = writer
 	case FormatNone:
 		compressor = fd
 	case FormatZstd:

internal/archive/archive_test.go

@@ -249,6 +249,12 @@ func TestExtractFormatLevel(t *testing.T) {
 			expectedFormat: FormatLzma,
 			expectedLevel:  LevelDefault,
 		},
+		{
+			name:           "lz4, fast",
+			in:             "lz4:fast",
+			expectedFormat: FormatLz4,
+			expectedLevel:  LevelFast,
+		},
 		{
 			name:           "none",
 			in:             "none",

internal/bootdeploy/bootdeploy.go

@@ -10,32 +10,32 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
 
 type BootDeploy struct {
 	inDir   string
 	outDir  string
-	ubootBoardname string
+	devinfo deviceinfo.DeviceInfo
 }
 
 // New returns a new BootDeploy, which then runs:
 //
 //	boot-deploy -d indir -o outDir
 //
-// ubootBoardname is used for copying in some u-boot files prior to running
-// boot-deploy. This is optional, passing an empty string is ok if this is not
-// needed.
-func New(inDir, outDir, ubootBoardname string) *BootDeploy {
+// devinfo is used to access some deviceinfo values, such as UbootBoardname
+// and GenerateSystemdBoot
+func New(inDir string, outDir string, devinfo deviceinfo.DeviceInfo) *BootDeploy {
 	return &BootDeploy{
 		inDir:   inDir,
 		outDir:  outDir,
-		ubootBoardname: ubootBoardname,
+		devinfo: devinfo,
 	}
 }
 
 func (b *BootDeploy) Run() error {
-
-	if err := copyUbootFiles(b.inDir, b.ubootBoardname); errors.Is(err, os.ErrNotExist) {
+	if err := copyUbootFiles(b.inDir, b.devinfo.UbootBoardname); errors.Is(err, os.ErrNotExist) {
 		log.Println("u-boot files copying skipped: ", err)
 	} else {
 		if err != nil {
@@ -43,15 +43,9 @@ func (b *BootDeploy) Run() error {
 		}
 	}
 
-	return bootDeploy(b.inDir, b.outDir)
-}
-
-func bootDeploy(workDir string, outDir string) error {
-	// boot-deploy expects the kernel to be in the same dir as initramfs.
-	// Assume that the kernel is in the output dir...
-	kernels, _ := filepath.Glob(filepath.Join(outDir, "vmlinuz*"))
-	if len(kernels) == 0 {
-		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
+	kernels, err := getKernelPath(b.outDir, b.devinfo.GenerateSystemdBoot == "true")
+	if err != nil {
+		return err
 	}
 
 	// Pick a kernel that does not have suffixes added by boot-deploy
@@ -71,7 +65,7 @@ func bootDeploy(workDir string, outDir string) error {
 	defer kernFd.Close()
 
 	kernFilename := path.Base(kernFile)
-	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
+	kernFileCopy, err := os.Create(filepath.Join(b.inDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -87,8 +81,8 @@ func bootDeploy(workDir string, outDir string) error {
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
 		"-k", kernFilename,
-		"-d", workDir,
-		"-o", outDir,
+		"-d", b.inDir,
+		"-o", b.outDir,
 		"initramfs-extra")
 
 	cmd.Stdout = os.Stdout
@@ -100,6 +94,25 @@ func bootDeploy(workDir string, outDir string) error {
 	return nil
 }
 
+func getKernelPath(outDir string, zboot bool) ([]string, error) {
+	var kernels []string
+	if zboot {
+		kernels, _ = filepath.Glob(filepath.Join(outDir, "linux.efi"))
+		if len(kernels) > 0 {
+			return kernels, nil
+		}
+		// else fallback to vmlinuz* below
+	}
+
+	kernFile := "vmlinuz*"
+	kernels, _ = filepath.Glob(filepath.Join(outDir, kernFile))
+	if len(kernels) == 0 {
+		return nil, errors.New("Unable to find any kernels at " + filepath.Join(outDir, kernFile))
+	}
+
+	return kernels, nil
+}
+
 // Copy copies the file at srcFile path to a new file at dstFile path
 func copy(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)

internal/filelist/hookdirs/hookdirs.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 )
@@ -44,6 +45,10 @@ func (h *HookDirs) List() (*filelist.FileList, error) {
 		s := bufio.NewScanner(f)
 		for s.Scan() {
 			dir := s.Text()
+			if len(dir) == 0 || strings.HasPrefix(dir, "#") {
+				continue
+			}
+
 			files.Add(dir, dir)
 		}
 	}

internal/filelist/hookfiles/hookfiles.go

@@ -58,7 +58,12 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		src, dest, has_dest := strings.Cut(s.Text(), ":")
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		src, dest, has_dest := strings.Cut(line, ":")
 
 		fFiles, err := misc.GetFiles([]string{src}, true)
 		if err != nil {

internal/filelist/initramfs/initramfs.go

@@ -9,6 +9,7 @@ import (
 // combining the output from them.
 type Initramfs struct {
 	features []filelist.FileLister
+	files    *filelist.FileList
 }
 
 // New returns a new Initramfs that generate a list of files based on the given
@@ -20,15 +21,18 @@ func New(features []filelist.FileLister) *Initramfs {
 }
 
 func (i *Initramfs) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
+	if i.files != nil {
+		return i.files, nil
+	}
+	i.files = filelist.NewFileList()
 
 	for _, f := range i.features {
 		list, err := f.List()
 		if err != nil {
 			return nil, err
 		}
-		files.Import(list)
+		i.files.Import(list)
 	}
 
-	return files, nil
+	return i.files, nil
 }

internal/filelist/modules/modules.go

@@ -16,22 +16,17 @@ import (
 )
 
 type Modules struct {
-	modulesList     []string
 	modulesListPath string
 }
 
-// New returns a new Modules that will use the given moduleto provide a list
-// of script files.
-func New(modulesList []string, modulesListPath string) *Modules {
+// New returns a new Modules that will read in lists of kernel modules in the given path.
+func New(modulesListPath string) *Modules {
 	return &Modules{
-		modulesList:     modulesList,
 		modulesListPath: modulesListPath,
 	}
 }
 
 func (m *Modules) List() (*filelist.FileList, error) {
-	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
-
 	kernVer, err := osutil.GetKernelVersion()
 	if err != nil {
 		return nil, err
@@ -54,21 +49,10 @@ func (m *Modules) List() (*filelist.FileList, error) {
 		files.Add(file, file)
 	}
 
-	// slurp up given list of modules
-	for _, module := range m.modulesList {
-		if modFilelist, err := getModule(module, modDir); err != nil {
-			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
-		} else {
-			for _, file := range modFilelist {
-				files.Add(file, file)
-			}
-		}
-	}
-
 	// slurp up modules from lists in modulesListPath
+	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
 	if err != nil {
-		log.Println("-- Unable to find dir, skipping...")
 		return files, nil
 	}
 	for _, file := range fileInfo {
@@ -94,6 +78,9 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
 		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
 		dir, file := filepath.Split(line)
 		if file == "" {
 			// item is a directory
@@ -126,8 +113,14 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 }
 
 func getModulesInDir(modPath string) (files []string, err error) {
-	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
+	err = filepath.Walk(modPath, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			// Unable to walk path
+			return err
+		}
+		// this assumes module names are in the format <name>.ko[.format],
+		// where ".format" (e.g. ".gz") is optional.
+		if !strings.Contains(".ko", path) {
 			return nil
 		}
 		files = append(files, path)
@@ -189,7 +182,12 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {
-		fields := strings.Fields(s.Text())
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		fields := strings.Fields(line)
 		if len(fields) == 0 {
 			continue
 		}

internal/filelist/osksdl/osksdl.go

@@ -1,158 +0,0 @@
-package osksdl
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-)
-
-type OskSdl struct {
-	mesaDriver string
-}
-
-// New returns a new HookScripts that will use the given path to provide a list
-// of script files.
-func New(mesaDriverName string) *OskSdl {
-	return &OskSdl{
-		mesaDriver: mesaDriverName,
-	}
-}
-
-// Get a list of files and their dependencies related to supporting rootfs full
-// disk (d)encryption
-func (s *OskSdl) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
-
-	if exists, err := misc.Exists("/usr/bin/osk-sdl"); !exists {
-		return files, nil
-	} else if err != nil {
-		return files, fmt.Errorf("received unexpected error when getting status for %q: %w", "/usr/bin/osk-sdl", err)
-	}
-
-	log.Println("- Including osk-sdl support")
-
-	confFiles := []string{
-		"/etc/osk.conf",
-		"/etc/ts.conf",
-		"/etc/pointercal",
-		"/etc/fb.modes",
-		"/etc/directfbrc",
-	}
-	confFileList, err := misc.GetFiles(confFiles, false)
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	}
-	for _, file := range confFileList {
-		files.Add(file, file)
-	}
-
-	// osk-sdl
-	oskFiles := []string{
-		"/usr/bin/osk-sdl",
-		"/sbin/cryptsetup",
-		"/usr/lib/libGL.so.1",
-	}
-	if oskFileList, err := misc.GetFiles(oskFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range oskFileList {
-			files.Add(file, file)
-		}
-	}
-
-	fontFile, err := getOskConfFontPath("/etc/osk.conf")
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
-	}
-	files.Add(fontFile, fontFile)
-
-	// Directfb
-	dfbFiles := []string{}
-	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			dfbFiles = append(dfbFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
-	}
-	if dfbFileList, err := misc.GetFiles(dfbFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range dfbFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// tslib
-	tslibFiles := []string{}
-	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			tslibFiles = append(tslibFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
-	}
-	libts, _ := filepath.Glob("/usr/lib/libts*")
-	tslibFiles = append(tslibFiles, libts...)
-	if tslibFileList, err := misc.GetFiles(tslibFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range tslibFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// mesa hw accel
-	if s.mesaDriver != "" {
-		mesaFiles := []string{
-			"/usr/lib/libEGL.so.1",
-			"/usr/lib/libGLESv2.so.2",
-			"/usr/lib/libgbm.so.1",
-			"/usr/lib/libudev.so.1",
-			"/usr/lib/xorg/modules/dri/" + s.mesaDriver + "_dri.so",
-		}
-		if mesaFileList, err := misc.GetFiles(mesaFiles, true); err != nil {
-			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-		} else {
-			for _, file := range mesaFileList {
-				files.Add(file, file)
-			}
-		}
-	}
-
-	return files, nil
-}
-
-func getOskConfFontPath(oskConfPath string) (string, error) {
-	var path string
-	f, err := os.Open(oskConfPath)
-	if err != nil {
-		return path, err
-	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		fields := strings.Fields(s.Text())
-		// "key = val" is 3 fields
-		if len(fields) > 2 && fields[0] == "keyboard-font" {
-			path = fields[2]
-		}
-	}
-	if exists, err := misc.Exists(path); !exists {
-		return path, fmt.Errorf("unable to find font: %s", path)
-	} else if err != nil {
-		return path, fmt.Errorf("received unexpected error when getting status for %q: %w", path, err)
-	}
-
-	return path, nil
-}

internal/misc/getfiles.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
+	"log"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 )
@@ -22,30 +24,54 @@ func GetFiles(list []string, required bool) (files []string, err error) {
 	return
 }
 
-func getFile(file string, required bool) (files []string, err error) {
-	// Expand glob expression
-	expanded, err := filepath.Glob(file)
+// This function doesn't handle globs, use getFile() instead.
+func getFileNormalized(file string, required bool) (files []string, err error) {
+	fileInfo, err := os.Stat(file)
+
+	// Trying some fallbacks...
 	if err != nil {
-		return
-	}
-	if len(expanded) > 0 && expanded[0] != file {
-		for _, path := range expanded {
-			if globFiles, err := getFile(path, required); err != nil {
-				return files, err
-			} else {
-				files = append(files, globFiles...)
-			}
-		}
-		return RemoveDuplicates(files), nil
+		type triedResult struct {
+			file string
+			err error
 		}
 
-	fileInfo, err := os.Stat(file)
-	if err != nil {
-		if required {
-			return files, fmt.Errorf("getFile: failed to stat file %q: %w", file, err)
+		triedFiles := make([]triedResult, 0, 1)
+
+		// Temporary fallback until alpine/pmOS usr-merge happened
+		// If a path starts with /bin or /sbin, also try /usr equivalent before giving up
+		if strings.HasPrefix(file, "/bin/") || strings.HasPrefix(file, "/sbin/") {
+			fileUsr := filepath.Join("/usr", file)
+			_, err := os.Stat(fileUsr);
+			if err == nil {
+				log.Printf("getFile: failed to find %q, but found it in %q. Please adjust the path.", file, fileUsr)
+				return getFileNormalized(fileUsr, required)
+			} else {
+				triedFiles = append(triedFiles, triedResult{fileUsr, err})
 			}
+		}
+
+		{
+			// Check if there is a Zstd-compressed version of the file
+			fileZstd := file + ".zst" // .zst is the extension used by linux-firmware
+			_, err := os.Stat(fileZstd);
+			if err == nil {
+				return getFileNormalized(fileZstd, required)
+			} else {
+				triedFiles = append(triedFiles, triedResult{fileZstd, err})
+			}
+		}
+		
+		// Failed to find anything
+		if required {
+			failStrings := make([]string, 0, 2)
+			for _, result := range triedFiles {
+				failStrings = append(failStrings, fmt.Sprintf("\n - also tried %q: %v", result.file, result.err))
+			}
+			return files, fmt.Errorf("getFile: failed to stat file %q: %v%q", file, err, strings.Join(failStrings, ""))
+		} else {
 			return files, nil
 		}
+	}
 
 	if fileInfo.IsDir() {
 		// Recurse over directory contents
@@ -83,6 +109,26 @@ func getFile(file string, required bool) (files []string, err error) {
 	return
 }
 
+func getFile(file string, required bool) (files []string, err error) {
+	// Expand glob expression
+	expanded, err := filepath.Glob(file)
+	if err != nil {
+		return
+	}
+	if len(expanded) > 0 && expanded[0] != file {
+		for _, path := range expanded {
+			if globFiles, err := getFile(path, required); err != nil {
+				return files, err
+			} else {
+				files = append(files, globFiles...)
+			}
+		}
+		return RemoveDuplicates(files), nil
+	}
+
+	return getFileNormalized(file, required)
+}
+
 func getDeps(file string, parents map[string]struct{}) (files []string, err error) {
 
 	if _, found := parents[file]; found {

internal/misc/misc.go

@@ -45,7 +45,7 @@ func RemoveDuplicates(in []string) (out []string) {
 //	defer misc.TimeFunc(time.Now(), "foo")
 func TimeFunc(start time.Time, name string) {
 	elapsed := time.Since(start)
-	log.Printf("%s completed in: %s", name, elapsed)
+	log.Printf("%s completed in: %.2fs", name, elapsed.Seconds())
 }
 
 // Exists tests if the given file/dir exists or not. Returns any errors related

pkgs/deviceinfo/deviceinfo.go

@@ -11,34 +11,42 @@ import (
 	"os"
 	"reflect"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type DeviceInfo struct {
 	InitfsCompression      string
 	InitfsExtraCompression string
-	MesaDriver             string
-	ModulesInitfs          string
 	UbootBoardname         string
+	GenerateSystemdBoot    string
 }
 
-func ReadDeviceinfo(file string) (DeviceInfo, error) {
-	var deviceinfo DeviceInfo
+// Reads the relevant entries from "file" into DeviceInfo struct
+// Any already-set entries will be overwriten if they are present
+// in "file"
+func (d *DeviceInfo) ReadDeviceinfo(file string) error {
+	if exists, err := misc.Exists(file); !exists {
+		return fmt.Errorf("%q not found, required by mkinitfs", file)
+	} else if err != nil {
+		return fmt.Errorf("unexpected error getting status for %q: %s", file, err)
+	}
 
 	fd, err := os.Open(file)
 	if err != nil {
-		return deviceinfo, err
+		return err
 	}
 	defer fd.Close()
 
-	if err := unmarshal(fd, &deviceinfo); err != nil {
-		return deviceinfo, err
+	if err := d.unmarshal(fd); err != nil {
+		return err
 	}
 
-	return deviceinfo, nil
+	return nil
 }
 
 // Unmarshals a deviceinfo into a DeviceInfo struct
-func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
+func (d *DeviceInfo) unmarshal(r io.Reader) error {
 	s := bufio.NewScanner(r)
 	for s.Scan() {
 		line := s.Text()
@@ -74,7 +82,7 @@ func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
 			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
 		}
 
-		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
+		field := reflect.ValueOf(d).Elem().FieldByName(fieldName)
 		if !field.IsValid() {
 			// an option that meets the deviceinfo "specification", but isn't
 			// one we care about in this module

pkgs/deviceinfo/deviceinfo_test.go

@@ -10,6 +10,28 @@ import (
 	"testing"
 )
 
+// Test ReadDeviceinfo and the logic of reading from multiple files
+func TestReadDeviceinfo(t *testing.T) {
+	compression_expected := "gz -9"
+
+	var devinfo DeviceInfo
+	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
+	if !strings.Contains(err.Error(), "required by mkinitfs") {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-first")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-msm")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	if devinfo.InitfsCompression != compression_expected {
+		t.Errorf("expected %q, got: %q", compression_expected, devinfo.InitfsCompression)
+	}
+}
+
 // Test conversion of name to DeviceInfo struct field format
 func TestNameToField(t *testing.T) {
 	tables := []struct {
@@ -18,9 +40,9 @@ func TestNameToField(t *testing.T) {
 	}{
 		{"deviceinfo_dtb", "Dtb"},
 		{"dtb", "Dtb"},
-		{"deviceinfo_modules_initfs", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression", "InitfsCompression"},
 		{"modules_initfs", "ModulesInitfs"},
-		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression___", "InitfsCompression"},
 		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
@@ -41,14 +63,12 @@ func TestUnmarshal(t *testing.T) {
 		in       string
 		expected string
 	}{
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"gzip:-9\"\n", "gzip:-9"},
 		// line with multiple '='
 		{"InitfsCompression", "deviceinfo_initfs_compression=zstd:--foo=1 -T0 --bar=bazz", "zstd:--foo=1 -T0 --bar=bazz"},
 		// empty option
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"\"\n", ""},
 		// line with comment at the end
-		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
 		{"", "# this is a comment!\n", ""},
 		// empty lines are fine
 		{"", "", ""},
@@ -58,7 +78,7 @@ func TestUnmarshal(t *testing.T) {
 	var d DeviceInfo
 	for _, table := range tables {
 		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
-		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
+		if err := d.unmarshal(strings.NewReader(table.in)); err != nil {
 			t.Errorf("%s received an unexpected err: ", err)
 		}
 

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -0,0 +1,2 @@
+deviceinfo_initfs_compression="gz -9"
+deviceinfo_mesa_driver="panfrost"

pkgs/deviceinfo/test_resources/deviceinfo-msm

@@ -0,0 +1 @@
+deviceinfo_mesa_driver="msm"