support building for a specific kernel version

Slightly rework things so that we can provide a kernel release/version
tag to mkinitfs and have it only operate on that, suffixing all the
relevant files and passing the info on to boot-deploy.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

Makefile

@@ -13,6 +13,11 @@ GOFLAGS?=
 LDFLAGS+=-s -w -X main.Version=$(VERSION)
 RM?=rm -f
 GOTEST=go test -count=1 -race
+DISABLE_GOGC?=
+
+ifeq ($(DISABLE_GOGC),1)
+	LDFLAGS+=-X main.DisableGC=true
+endif
 
 GOSRC!=find * -name '*.go'
 GOSRC+=go.mod go.sum

cmd/mkinitfs/main.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"strings"
 	"time"
 
@@ -20,7 +21,6 @@ import (
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/hookscripts"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/initramfs"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
@@ -28,12 +28,19 @@ import (
 
 // set at build time
 var Version string
+var DisableGC string
 
 func main() {
+	// To allow working around silly GC-related issues, like https://gitlab.com/qemu-project/qemu/-/issues/2560
+	if strings.ToLower(DisableGC) == "true" {
+		debug.SetGCPercent(-1)
+	}
+
 	retCode := 0
 	defer func() { os.Exit(retCode) }()
 
 	outDir := flag.String("d", "/boot", "Directory to output initfs(-extra) and other boot files")
+	kernVerArg := flag.String("k", "guess", "Kernel version to run for")
 
 	var showVersion bool
 	flag.BoolVar(&showVersion, "version", false, "Print version and quit.")
@@ -42,6 +49,8 @@ func main() {
 	flag.BoolVar(&disableBootDeploy, "no-bootdeploy", false, "Disable running 'boot-deploy' after generating archives.")
 	flag.Parse()
 
+	kernVer := *kernVerArg
+
 	if showVersion {
 		fmt.Printf("%s - %s\n", filepath.Base(os.Args[0]), Version)
 		return
@@ -49,29 +58,27 @@ func main() {
 
 	log.Default().SetFlags(log.Lmicroseconds)
 
-	deviceinfoFile := "/etc/deviceinfo"
+	var devinfo deviceinfo.DeviceInfo
-	if exists, err := misc.Exists(deviceinfoFile); !exists {
+	deverr_usr := devinfo.ReadDeviceinfo("/usr/share/deviceinfo/deviceinfo")
-		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
+	deverr_etc := devinfo.ReadDeviceinfo("/etc/deviceinfo")
-		return
+	if deverr_etc != nil && deverr_usr != nil {
-	} else if err != nil {
+		log.Println("Error reading deviceinfo")
-		retCode = 1
+		log.Println("\t/usr/share/deviceinfo/deviceinfo:", deverr_usr)
-		log.Printf("received unexpected error when getting status for %q: %s", deviceinfoFile, err)
+		log.Println("\t/etc/deviceinfo:", deverr_etc)
-	}
-
-	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
-	if err != nil {
-		log.Println(err)
 		retCode = 1
 		return
 	}
 
 	defer misc.TimeFunc(time.Now(), "mkinitfs")
 
-	kernVer, err := osutil.GetKernelVersion()
+	if kernVer == "guess" {
-	if err != nil {
+		_kernVer, err := osutil.GetKernelVersion()
-		log.Println(err)
+		if err != nil {
-		retCode = 1
+			log.Println(err)
-		return
+			retCode = 1
+			return
+		}
+		kernVer = _kernVer
 	}
 
 	// temporary working dir
@@ -85,52 +92,98 @@ func main() {
 	defer func() {
 		e := os.RemoveAll(workDir)
 		if e != nil && err == nil {
-			err = e
+			log.Println(e)
-			retCode = 1
+			log.Println("unable to remove temporary work directory")
 		}
 	}()
 
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
+	//
+	// initramfs
+	//
 	// deviceinfo.InitfsCompression needs a little more post-processing
 	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	log.Printf("== Generating %s ==\n", "initramfs")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start := time.Now()
+	initramfsAr := archive.New(compressionFormat, compressionLevel)
+	initfs := initramfs.New([]filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
 		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
 		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
-		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
+		modules.New("/usr/share/mkinitfs/modules", kernVer),
-		modules.New([]string{}, "/etc/mkinitfs/modules"),
+		modules.New("/etc/mkinitfs/modules", kernVer),
-	}); err != nil {
+	})
+	initfsExtra := initramfs.New([]filelist.FileLister{
+		hookfiles.New("/usr/share/mkinitfs/files-extra"),
+		hookfiles.New("/etc/mkinitfs/files-extra"),
+		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
+		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
+		modules.New("/usr/share/mkinitfs/modules-extra", kernVer),
+		modules.New("/etc/mkinitfs/modules-extra", kernVer),
+	})
+
+	if err := initramfsAr.AddItems(initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
 
-	// deviceinfo.InitfsExtraCompression needs a little more post-processing
+	// Include initramfs-extra files in the initramfs if not making a separate
-	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
+	// archive
-	if err := generateArchive("initramfs-extra", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	if !devinfo.CreateInitfsExtra {
-		hookfiles.New("/usr/share/mkinitfs/files-extra"),
+		if err := initramfsAr.AddItems(initfsExtra); err != nil {
-		hookfiles.New("/etc/mkinitfs/files-extra"),
+			log.Println(err)
-		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
+			log.Println("failed to generate: ", "initramfs")
-		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
+			retCode = 1
-		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
+			return
-		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
+		}
-		osksdl.New(devinfo.MesaDriver),
+	}
-	}); err != nil {
+
+	if err := initramfsAr.Write(filepath.Join(workDir, fmt.Sprintf("initramfs-%s", kernVer)), os.FileMode(0644)); err != nil {
 		log.Println(err)
-		log.Println("failed to generate: ", "initramfs-extra")
+		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
+	misc.TimeFunc(start, "initramfs")
+
+	if devinfo.CreateInitfsExtra {
+		//
+		// initramfs-extra
+		//
+		// deviceinfo.InitfsExtraCompression needs a little more post-processing
+		compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
+		log.Printf("== Generating %s ==\n", "initramfs-extra")
+		log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+		start = time.Now()
+		initramfsExtraAr := archive.New(compressionFormat, compressionLevel)
+		if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
+			log.Println(err)
+			log.Println("failed to generate: ", "initramfs-extra")
+			retCode = 1
+			return
+		}
+		if err := initramfsExtraAr.Write(filepath.Join(workDir, "initramfs-extra"), os.FileMode(0644)); err != nil {
+			log.Println(err)
+			log.Println("failed to generate: ", "initramfs-extra")
+			retCode = 1
+			return
+		}
+		misc.TimeFunc(start, "initramfs-extra")
+	}
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
-		if err := bootDeploy(workDir, *outDir, devinfo.UbootBoardname); err != nil {
+		if err := bootDeploy(workDir, *outDir, devinfo, kernVer); err != nil {
 			log.Println(err)
 			log.Println("boot-deploy failed")
 			retCode = 1
@@ -139,30 +192,10 @@ func main() {
 	}
 }
 
-func bootDeploy(workDir, outDir, ubootBoardname string) error {
+func bootDeploy(workDir string, outDir string, devinfo deviceinfo.DeviceInfo, kernVer string) error {
 	log.Print("== Using boot-deploy to finalize/install files ==")
 	defer misc.TimeFunc(time.Now(), "boot-deploy")
 
-	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
+	bd := bootdeploy.New(workDir, outDir, devinfo, kernVer)
 	return bd.Run()
 }
-
-func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
-	log.Printf("== Generating %s ==\n", name)
-	log.Printf("- Using compression format %s with level %q\n", format, level)
-
-	defer misc.TimeFunc(time.Now(), name)
-	a := archive.New(format, level)
-
-	fs := initramfs.New(features)
-	if err := a.AddItems(fs); err != nil {
-		return err
-	}
-
-	log.Println("- Writing and verifying archive: ", name)
-	if err := a.Write(filepath.Join(path, name), os.FileMode(0644)); err != nil {
-		return err
-	}
-
-	return nil
-}

doc/mkinitfs.1.scd

@@ -38,13 +38,14 @@ Design goals of this project are:
 The canonical deviceinfo "specification" is at
 https://wiki.postmarketos.org/wiki/Deviceinfo_reference
 
-mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
+mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
+*/etc/deviceinfo*, in that order. The following variables
 are *required* by mkinitfs:
 
+	- deviceinfo_create_initfs_extra
+	- deviceinfo_generate_systemd_boot
 	- deviceinfo_initfs_compression
 	- deviceinfo_initfs_extra_compression
-	- deviceinfo_mesa_driver
-	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
 
 It is a design goal to keep the number of required variables from deviceinfo to
@@ -54,6 +55,36 @@ a bare minimum, and to require only variables that don't hold lists of things.
 necessary tools to extract the configured archive format are in the initramfs
 archive.
 
+# ARCHIVE COMPRESSION
+
+Archive compression parameters are specified in the
+*deviceinfo_initfs_compression* and *deviceinfo_initfs_extra_compression*
+deviceinfo variables. Their values do not have to match, but special
+consideration should be taken since some formats may require additional kernel
+options or tools in the initramfs to support it.
+
+Supported compression *formats* for mkinitfs are:
+
+	- gzip
+	- lz4
+	- lzma
+	- none
+	- zstd
+
+Supported compression *levels* for mkinitfs:
+
+	- best
+	- default
+	- fast
+
+The value of these variables follows this syntax: *<format>:<level>*. For
+example, *zstd* with the *fast* compression level would be:
+*deviceinfo_initfs_compression="zstd:fast"*
+
+Defaults to *gzip* and *default* for both archives if format and/or level is
+unsupported or omitted.
+
+
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and
@@ -63,7 +94,7 @@ it are for constructing the initramfs archive.
 
 Configuration under */usr/share/mkinitfs* is intended to be managed by
 distributions, while configuration under */etc/mkinitfs* is for users to
-create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*. 
+create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*.
 
 ## /usr/share/mkinitfs/files, /etc/mkinitfs/files
 ## /usr/share/mkinitfs/files-extra, /etc/mkinitfs/files-extra
@@ -99,6 +130,9 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	path(s) under the relevant directory in */etc/mkinitfs*, and changing
 	the destination path.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/hooks, /etc/mkinitfs/hooks
 ## /usr/share/mkinitfs/hooks-extra*, /etc/mkinitfs/hooks-extra
 
@@ -113,19 +147,25 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 ## /usr/share/mkinitfs/modules, /etc/mkinitfs/modules
 ## /usr/share/mkinitfs/modules-extra, /etc/mkinitfs/modules-extra
 
-	Files with the *.modules* extention in these directories are lists of
+	Files with the *.modules* extension in these directories are lists of
 	kernel modules to include in the initramfs. Individual modules and
 	directories can be listed in the files here. Globbing is also supported.
 
 	Modules are installed in the initramfs archive under the same path they
 	exist on the system where mkinitfs is executed.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/dirs, /etc/mkinitfs/dirs
 
 	Files with the *.dirs* extension in these directories are lists of
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 # BOOT-DEPLOY
 
 After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
@@ -136,7 +176,7 @@ search for the app. The following commandline options are passed to it:
 		Currently this is hardcoded to be "initramfs"
 
 	*-k* <kernel filename>
-	
+
 	*-d* <work directory>
 
 		Path to the directory containing the build artifacts from mkinitfs.

go.mod

@@ -5,6 +5,15 @@ go 1.20
 require (
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.15.12
+	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/ulikunitz/xz v0.5.10
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
+	golang.org/x/sys v0.18.0
+)
+
+require (
+	github.com/mvdan/sh v2.6.4+incompatible // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
+	mvdan.cc/sh v2.6.4+incompatible // indirect
 )

go.sum

@@ -2,7 +2,21 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RS
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
 github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
 github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/mvdan/sh v2.6.4+incompatible h1:D4oEWW0J8cL7zeQkrXw76IAYXF0mJfDaBwjgzmKb6zs=
+github.com/mvdan/sh v2.6.4+incompatible/go.mod h1:kipHzrJQZEDCMTNRVRAlMMFjqHEYrthfIlFkJSrmDZE=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+mvdan.cc/sh v2.6.4+incompatible h1:eD6tDeh0pw+/TOTI1BBEryZ02rD2nMcFsgcvde7jffM=
+mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=

internal/archive/archive.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/cavaliercoder/go-cpio"
 	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/ulikunitz/xz"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
@@ -28,6 +29,7 @@ type CompressFormat string
 const (
 	FormatGzip CompressFormat = "gzip"
 	FormatLzma CompressFormat = "lzma"
+	FormatLz4  CompressFormat = "lz4"
 	FormatZstd CompressFormat = "zstd"
 	FormatNone CompressFormat = "none"
 )
@@ -104,6 +106,7 @@ func ExtractFormatLevel(s string) (format CompressFormat, level CompressLevel) {
 	case FormatLzma:
 		log.Println("Format lzma doesn't support a compression level, using default settings")
 		level = LevelDefault
+	case FormatLz4:
 	case FormatNone:
 	case FormatZstd:
 	default:
@@ -200,9 +203,44 @@ func (archive *Archive) AddItems(flister filelist.FileLister) error {
 	return nil
 }
 
+// AddItemsExclude is like AddItems, but takes a second FileLister that lists
+// items that should not be added to the archive from the first FileLister
+func (archive *Archive) AddItemsExclude(flister filelist.FileLister, exclude filelist.FileLister) error {
+	list, err := flister.List()
+	if err != nil {
+		return err
+	}
+
+	excludeList, err := exclude.List()
+	if err != nil {
+		return err
+	}
+
+	for i := range list.IterItems() {
+		dest, found := excludeList.Get(i.Source)
+
+		if found {
+			if i.Dest != dest {
+				found = false
+			}
+		}
+
+		if !found {
+			if err := archive.AddItem(i.Source, i.Dest); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // Adds the given file or directory at "source" to the archive at "dest"
 func (archive *Archive) AddItem(source string, dest string) error {
-
+	if osutil.HasMergedUsr() {
+		source = osutil.MergeUsr(source)
+		dest = osutil.MergeUsr(dest)
+	}
 	sourceStat, err := os.Lstat(source)
 	if err != nil {
 		e, ok := err.(*os.PathError)
@@ -213,6 +251,12 @@ func (archive *Archive) AddItem(source string, dest string) error {
 		return fmt.Errorf("AddItem: failed to get stat for %q: %w", source, err)
 	}
 
+	// A symlink to a directory doesn't have the os.ModeDir bit set, so we need
+	// to check if it's a symlink first
+	if sourceStat.Mode()&os.ModeSymlink != 0 {
+		return archive.addSymlink(source, dest)
+	}
+
 	if sourceStat.Mode()&os.ModeDir != 0 {
 		return archive.addDir(dest)
 	}
@@ -220,6 +264,45 @@ func (archive *Archive) AddItem(source string, dest string) error {
 	return archive.addFile(source, dest)
 }
 
+func (archive *Archive) addSymlink(source string, dest string) error {
+	target, err := os.Readlink(source)
+	if err != nil {
+		log.Print("addSymlink: failed to get symlink target for: ", source)
+		return err
+	}
+
+	// Make sure we pick up the symlink target too
+	targetAbs := target
+	if filepath.Dir(target) == "." {
+		// relative symlink, make it absolute so we can add the target to the archive
+		targetAbs = filepath.Join(filepath.Dir(source), target)
+	}
+
+	if !filepath.IsAbs(targetAbs) {
+		targetAbs, err = osutil.RelativeSymlinkTargetToDir(targetAbs, filepath.Dir(source))
+		if err != nil {
+			return err
+		}
+	}
+
+	archive.AddItem(targetAbs, targetAbs)
+
+	// Now add the symlink itself
+	destFilename := strings.TrimPrefix(dest, "/")
+
+	archive.items.add(archiveItem{
+		sourcePath: source,
+		header: &cpio.Header{
+			Name:     destFilename,
+			Linkname: target,
+			Mode:     0644 | cpio.ModeSymlink,
+			Size:     int64(len(target)),
+		},
+	})
+
+	return nil
+}
+
 func (archive *Archive) addFile(source string, dest string) error {
 	if err := archive.addDir(filepath.Dir(dest)); err != nil {
 		return err
@@ -231,42 +314,6 @@ func (archive *Archive) addFile(source string, dest string) error {
 		return err
 	}
 
-	// Symlink: write symlink to archive then set 'file' to link target
-	if sourceStat.Mode()&os.ModeSymlink != 0 {
-		// log.Printf("File %q is a symlink", file)
-		target, err := os.Readlink(source)
-		if err != nil {
-			log.Print("addFile: failed to get symlink target: ", source)
-			return err
-		}
-
-		destFilename := strings.TrimPrefix(dest, "/")
-
-		archive.items.add(archiveItem{
-			sourcePath: source,
-			header: &cpio.Header{
-				Name:     destFilename,
-				Linkname: target,
-				Mode:     0644 | cpio.ModeSymlink,
-				Size:     int64(len(target)),
-				// Checksum: 1,
-			},
-		})
-
-		if filepath.Dir(target) == "." {
-			target = filepath.Join(filepath.Dir(source), target)
-		}
-		// make sure target is an absolute path
-		if !filepath.IsAbs(target) {
-			target, err = osutil.RelativeSymlinkTargetToDir(target, filepath.Dir(source))
-			if err != nil {
-				return err
-			}
-		}
-		err = archive.addFile(target, target)
-		return err
-	}
-
 	destFilename := strings.TrimPrefix(dest, "/")
 
 	archive.items.add(archiveItem{
@@ -316,6 +363,23 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 		if err != nil {
 			return err
 		}
+	case FormatLz4:
+		// The default compression for the lz4 library is Fast, and
+		// they don't define a Default level otherwise
+		level := lz4.Fast
+		switch archive.compress_level {
+		case LevelBest:
+			level = lz4.Level9
+		case LevelFast:
+			level = lz4.Fast
+		}
+
+		var writer = lz4.NewWriter(fd)
+		err = writer.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(level))
+		if err != nil {
+			return err
+		}
+		compressor = writer
 	case FormatNone:
 		compressor = fd
 	case FormatZstd:
@@ -352,6 +416,13 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 }
 
 func (archive *Archive) writeCpio() error {
+	// Just in case
+	if osutil.HasMergedUsr() {
+		archive.addSymlink("/bin", "/bin")
+		archive.addSymlink("/sbin", "/sbin")
+		archive.addSymlink("/lib", "/lib")
+		archive.addSymlink("/usr/sbin", "/usr/sbin")
+	}
 	// having a transient function for actually adding files to the archive
 	// allows the deferred fd.close to run after every copy and prevent having
 	// tons of open file handles until the copying is all done
@@ -366,19 +437,19 @@ func (archive *Archive) writeCpio() error {
 			if header.Mode.IsRegular() {
 				fd, err := os.Open(source)
 				if err != nil {
-					return fmt.Errorf("archive.writeCpio: uname to open file %q, %w", source, err)
+					return fmt.Errorf("archive.writeCpio: Unable to open file %q, %w", source, err)
 				}
 				defer fd.Close()
 				if _, err := io.Copy(archive.cpioWriter, fd); err != nil {
-					return fmt.Errorf("archive.writeCpio: unable to write out archive: %w", err)
+					return fmt.Errorf("archive.writeCpio: Couldn't process %q: %w", source, err)
 				}
 			} else if header.Linkname != "" {
 				// the contents of a symlink is just need the link name
 				if _, err := archive.cpioWriter.Write([]byte(header.Linkname)); err != nil {
-					return fmt.Errorf("archive.writeCpio: unable to write out symlink: %w", err)
+					return fmt.Errorf("archive.writeCpio: unable to write out symlink: %q -> %q: %w", source, header.Linkname, err)
 				}
 			} else {
-				return fmt.Errorf("archive.writeCpio: unknown type for file: %s", source)
+				return fmt.Errorf("archive.writeCpio: unknown type for file: %q: %d", source, header.Mode)
 			}
 		}
 

internal/archive/archive_test.go

@@ -249,6 +249,12 @@ func TestExtractFormatLevel(t *testing.T) {
 			expectedFormat: FormatLzma,
 			expectedLevel:  LevelDefault,
 		},
+		{
+			name:           "lz4, fast",
+			in:             "lz4:fast",
+			expectedFormat: FormatLz4,
+			expectedLevel:  LevelFast,
+		},
 		{
 			name:           "none",
 			in:             "none",

internal/bootdeploy/bootdeploy.go

@@ -10,32 +10,34 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
 
 type BootDeploy struct {
-	inDir          string
+	inDir   string
-	outDir         string
+	outDir  string
-	ubootBoardname string
+	devinfo deviceinfo.DeviceInfo
+	kernVer string
 }
 
 // New returns a new BootDeploy, which then runs:
 //
 //	boot-deploy -d indir -o outDir
 //
-// ubootBoardname is used for copying in some u-boot files prior to running
+// devinfo is used to access some deviceinfo values, such as UbootBoardname
-// boot-deploy. This is optional, passing an empty string is ok if this is not
+// and GenerateSystemdBoot
-// needed.
+func New(inDir string, outDir string, devinfo deviceinfo.DeviceInfo, kernVer string) *BootDeploy {
-func New(inDir, outDir, ubootBoardname string) *BootDeploy {
 	return &BootDeploy{
-		inDir:          inDir,
+		inDir:   inDir,
-		outDir:         outDir,
+		outDir:  outDir,
-		ubootBoardname: ubootBoardname,
+		devinfo: devinfo,
+		kernVer: kernVer,
 	}
 }
 
 func (b *BootDeploy) Run() error {
-
+	if err := copyUbootFiles(b.inDir, b.devinfo.UbootBoardname); errors.Is(err, os.ErrNotExist) {
-	if err := copyUbootFiles(b.inDir, b.ubootBoardname); errors.Is(err, os.ErrNotExist) {
 		log.Println("u-boot files copying skipped: ", err)
 	} else {
 		if err != nil {
@@ -43,16 +45,11 @@ func (b *BootDeploy) Run() error {
 		}
 	}
 
-	return bootDeploy(b.inDir, b.outDir)
+	kernels, err := getKernelPath(b.outDir, b.kernVer, b.devinfo.GenerateSystemdBoot == "true")
-}
+	if err != nil {
-
+		return err
-func bootDeploy(workDir string, outDir string) error {
-	// boot-deploy expects the kernel to be in the same dir as initramfs.
-	// Assume that the kernel is in the output dir...
-	kernels, _ := filepath.Glob(filepath.Join(outDir, "vmlinuz*"))
-	if len(kernels) == 0 {
-		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
 	}
+	println(fmt.Sprintf("kernels: %v\n", kernels))
 
 	// Pick a kernel that does not have suffixes added by boot-deploy
 	var kernFile string
@@ -71,7 +68,7 @@ func bootDeploy(workDir string, outDir string) error {
 	defer kernFd.Close()
 
 	kernFilename := path.Base(kernFile)
-	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
+	kernFileCopy, err := os.Create(filepath.Join(b.inDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -84,12 +81,19 @@ func bootDeploy(workDir string, outDir string) error {
 	}
 
 	// boot-deploy -i initramfs -k vmlinuz-postmarketos-rockchip -d /tmp/cpio -o /tmp/foo initramfs-extra
-	cmd := exec.Command("boot-deploy",
+	args := []string{
-		"-i", "initramfs",
+		"-i", fmt.Sprintf("initramfs-%s", b.kernVer),
 		"-k", kernFilename,
-		"-d", workDir,
+		"-v", b.kernVer,
-		"-o", outDir,
+		"-d", b.inDir,
-		"initramfs-extra")
+		"-o", b.outDir,
+	}
+
+	if b.devinfo.CreateInitfsExtra {
+		args = append(args, "initramfs-extra")
+	}
+	println(fmt.Sprintf("Calling boot-deply with args: %v\n", args))
+	cmd := exec.Command("boot-deploy", args...)
 
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -100,6 +104,25 @@ func bootDeploy(workDir string, outDir string) error {
 	return nil
 }
 
+func getKernelPath(outDir string, kernVer string, zboot bool) ([]string, error) {
+	var kernels []string
+	if zboot {
+		kernels, _ = filepath.Glob(filepath.Join(outDir, fmt.Sprintf("linux-%s.efi", kernVer)))
+		if len(kernels) > 0 {
+			return kernels, nil
+		}
+		// else fallback to vmlinuz* below
+	}
+
+	kernFile := fmt.Sprintf("vmlinuz-%s", kernVer)
+	kernels, _ = filepath.Glob(filepath.Join(outDir, kernFile))
+	if len(kernels) == 0 {
+		return nil, errors.New("Unable to find any kernels at " + filepath.Join(outDir, kernFile) + " or " + filepath.Join(outDir, fmt.Sprintf("linux-%s.efi", kernVer)))
+	}
+
+	return kernels, nil
+}
+
 // Copy copies the file at srcFile path to a new file at dstFile path
 func copy(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)

internal/filelist/hookdirs/hookdirs.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 )
@@ -43,7 +44,11 @@ func (h *HookDirs) List() (*filelist.FileList, error) {
 
 		s := bufio.NewScanner(f)
 		for s.Scan() {
-			dir := s.Text()
+			dir := strings.TrimSpace(s.Text())
+			if len(dir) == 0 || strings.HasPrefix(dir, "#") {
+				continue
+			}
+
 			files.Add(dir, dir)
 		}
 	}

internal/filelist/hookfiles/hookfiles.go

@@ -11,6 +11,7 @@ import (
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 )
 
 type HookFiles struct {
@@ -58,7 +59,15 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		src, dest, has_dest := strings.Cut(s.Text(), ":")
+		line := strings.TrimSpace(s.Text())
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		src, dest, has_dest := strings.Cut(line, ":")
+		if osutil.HasMergedUsr() {
+			src = osutil.MergeUsr(src)
+		}
 
 		fFiles, err := misc.GetFiles([]string{src}, true)
 		if err != nil {

internal/filelist/initramfs/initramfs.go

@@ -9,6 +9,7 @@ import (
 // combining the output from them.
 type Initramfs struct {
 	features []filelist.FileLister
+	files    *filelist.FileList
 }
 
 // New returns a new Initramfs that generate a list of files based on the given
@@ -20,15 +21,18 @@ func New(features []filelist.FileLister) *Initramfs {
 }
 
 func (i *Initramfs) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
+	if i.files != nil {
+		return i.files, nil
+	}
+	i.files = filelist.NewFileList()
 
 	for _, f := range i.features {
 		list, err := f.List()
 		if err != nil {
 			return nil, err
 		}
-		files.Import(list)
+		i.files.Import(list)
 	}
 
-	return files, nil
+	return i.files, nil
 }

internal/filelist/modules/modules.go

@@ -12,32 +12,31 @@ import (
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 )
 
 type Modules struct {
 	modulesListPath string
-	modulesList     []string
+	kernVer         string
 }
 
-// New returns a new Modules that will use the given moduleto provide a list
+// New returns a new Modules that will read in lists of kernel modules in the given path.
-// of script files.
+func New(modulesListPath string, kernVer string) *Modules {
-func New(modulesList []string, modulesListPath string) *Modules {
 	return &Modules{
-		modulesList:     modulesList,
 		modulesListPath: modulesListPath,
+		kernVer:         kernVer,
 	}
 }
 
 func (m *Modules) List() (*filelist.FileList, error) {
-	kernVer, err := osutil.GetKernelVersion()
+	files := filelist.NewFileList()
-	if err != nil {
+	libDir := "/usr/lib/modules"
-		return nil, err
+	if exists, err := misc.Exists(libDir); !exists {
+		libDir = "/lib/modules"
+	} else if err != nil {
+		return nil, fmt.Errorf("received unexpected error when getting status for %q: %w", libDir, err)
 	}
 
-	files := filelist.NewFileList()
+	modDir := filepath.Join(libDir, m.kernVer)
-
-	modDir := filepath.Join("/lib/modules", kernVer)
 	if exists, err := misc.Exists(modDir); !exists {
 		// dir /lib/modules/<kernel> if kernel built without module support, so just print a message
 		log.Printf("-- kernel module directory not found: %q, not including modules", modDir)
@@ -52,20 +51,6 @@ func (m *Modules) List() (*filelist.FileList, error) {
 		files.Add(file, file)
 	}
 
-	// slurp up given list of modules
-	if len(m.modulesList) > 0 {
-		log.Printf("-- Including kernel modules from deviceinfo")
-		for _, module := range m.modulesList {
-			if modFilelist, err := getModule(module, modDir); err != nil {
-				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
-			} else {
-				for _, file := range modFilelist {
-					files.Add(file, file)
-				}
-			}
-		}
-	}
-
 	// slurp up modules from lists in modulesListPath
 	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
@@ -94,7 +79,10 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 	files := filelist.NewFileList()
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		line := s.Text()
+		line := strings.TrimSpace(s.Text())
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
 		dir, file := filepath.Split(line)
 		if file == "" {
 			// item is a directory
@@ -111,8 +99,8 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 			}
 		} else if dir == "" {
 			// item is a module name
-			if modFilelist, err := getModule(s.Text(), modDir); err != nil {
+			if modFilelist, err := getModule(line, modDir); err != nil {
-				return nil, fmt.Errorf("unable to get module file %q: %w", s.Text(), err)
+				return nil, fmt.Errorf("unable to get module file %q: %w", line, err)
 			} else {
 				for _, file := range modFilelist {
 					files.Add(file, file)
@@ -132,7 +120,9 @@ func getModulesInDir(modPath string) (files []string, err error) {
 			// Unable to walk path
 			return err
 		}
-		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
+		// this assumes module names are in the format <name>.ko[.format],
+		// where ".format" (e.g. ".gz") is optional.
+		if !strings.Contains(".ko", path) {
 			return nil
 		}
 		files = append(files, path)
@@ -194,7 +184,12 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {
-		fields := strings.Fields(s.Text())
+		line := strings.TrimSpace(s.Text())
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		fields := strings.Fields(line)
 		if len(fields) == 0 {
 			continue
 		}

internal/filelist/modules/modules_test.go

@@ -18,6 +18,7 @@ func TestStripExts(t *testing.T) {
 		{"another_file", "another_file"},
 		{"a.b.c.d.e.f.g.h.i", "a"},
 		{"virtio_blk.ko", "virtio_blk"},
+		{"virtio_blk.ko ", "virtio_blk"},
 	}
 	for _, table := range tables {
 		out := stripExts(table.in)

internal/filelist/osksdl/osksdl.go

@@ -1,158 +0,0 @@
-package osksdl
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-)
-
-type OskSdl struct {
-	mesaDriver string
-}
-
-// New returns a new HookScripts that will use the given path to provide a list
-// of script files.
-func New(mesaDriverName string) *OskSdl {
-	return &OskSdl{
-		mesaDriver: mesaDriverName,
-	}
-}
-
-// Get a list of files and their dependencies related to supporting rootfs full
-// disk (d)encryption
-func (s *OskSdl) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
-
-	if exists, err := misc.Exists("/usr/bin/osk-sdl"); !exists {
-		return files, nil
-	} else if err != nil {
-		return files, fmt.Errorf("received unexpected error when getting status for %q: %w", "/usr/bin/osk-sdl", err)
-	}
-
-	log.Println("- Including osk-sdl support")
-
-	confFiles := []string{
-		"/etc/osk.conf",
-		"/etc/ts.conf",
-		"/etc/pointercal",
-		"/etc/fb.modes",
-		"/etc/directfbrc",
-	}
-	confFileList, err := misc.GetFiles(confFiles, false)
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	}
-	for _, file := range confFileList {
-		files.Add(file, file)
-	}
-
-	// osk-sdl
-	oskFiles := []string{
-		"/usr/bin/osk-sdl",
-		"/sbin/cryptsetup",
-		"/usr/lib/libGL.so.1",
-	}
-	if oskFileList, err := misc.GetFiles(oskFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range oskFileList {
-			files.Add(file, file)
-		}
-	}
-
-	fontFile, err := getOskConfFontPath("/etc/osk.conf")
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
-	}
-	files.Add(fontFile, fontFile)
-
-	// Directfb
-	dfbFiles := []string{}
-	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			dfbFiles = append(dfbFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
-	}
-	if dfbFileList, err := misc.GetFiles(dfbFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range dfbFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// tslib
-	tslibFiles := []string{}
-	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			tslibFiles = append(tslibFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
-	}
-	libts, _ := filepath.Glob("/usr/lib/libts*")
-	tslibFiles = append(tslibFiles, libts...)
-	if tslibFileList, err := misc.GetFiles(tslibFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range tslibFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// mesa hw accel
-	if s.mesaDriver != "" {
-		mesaFiles := []string{
-			"/usr/lib/libEGL.so.1",
-			"/usr/lib/libGLESv2.so.2",
-			"/usr/lib/libgbm.so.1",
-			"/usr/lib/libudev.so.1",
-			"/usr/lib/xorg/modules/dri/" + s.mesaDriver + "_dri.so",
-		}
-		if mesaFileList, err := misc.GetFiles(mesaFiles, true); err != nil {
-			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-		} else {
-			for _, file := range mesaFileList {
-				files.Add(file, file)
-			}
-		}
-	}
-
-	return files, nil
-}
-
-func getOskConfFontPath(oskConfPath string) (string, error) {
-	var path string
-	f, err := os.Open(oskConfPath)
-	if err != nil {
-		return path, err
-	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		fields := strings.Fields(s.Text())
-		// "key = val" is 3 fields
-		if len(fields) > 2 && fields[0] == "keyboard-font" {
-			path = fields[2]
-		}
-	}
-	if exists, err := misc.Exists(path); !exists {
-		return path, fmt.Errorf("unable to find font: %s", path)
-	} else if err != nil {
-		return path, fmt.Errorf("received unexpected error when getting status for %q: %w", path, err)
-	}
-
-	return path, nil
-}

internal/misc/getfiles.go

@@ -41,10 +41,22 @@ func getFile(file string, required bool) (files []string, err error) {
 
 	fileInfo, err := os.Stat(file)
 	if err != nil {
-		if required {
+		// Check if there is a Zstd-compressed version of the file
-			return files, fmt.Errorf("getFile: failed to stat file %q: %w", file, err)
+		fileZstd := file + ".zst" // .zst is the extension used by linux-firmware
+		fileInfoZstd, errZstd := os.Stat(fileZstd)
+
+		if errZstd == nil {
+			file = fileZstd
+			fileInfo = fileInfoZstd
+			// Unset nil so we don't retain the error from the os.Stat call for the uncompressed version.
+			err = nil
+		} else {
+			if required {
+				return files, fmt.Errorf("getFile: failed to stat file %q: %w (also tried %q: %w)", file, err, fileZstd, errZstd)
+			}
+
+			return files, nil
 		}
-		return files, nil
 	}
 
 	if fileInfo.IsDir() {
@@ -108,6 +120,7 @@ func getDeps(file string, parents map[string]struct{}) (files []string, err erro
 		"/usr/lib",
 		"/lib",
 		"/usr/lib/expect*",
+		"/usr/lib/systemd",
 	}
 
 	for _, lib := range libs {

internal/osutil/osutil.go

@@ -10,6 +10,39 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+// Try to guess whether the system has merged dirs under /usr
+func HasMergedUsr() bool {
+	for _, dir := range []string{"/bin", "/lib"} {
+		stat, err := os.Lstat(dir)
+		if err != nil {
+			// TODO: probably because the dir doesn't exist... so
+			// should we assume that it's because the system has some weird
+			// implementation of "merge /usr"?
+			return true
+		} else if stat.Mode()&os.ModeSymlink == 0 {
+			// Not a symlink, so must not be merged /usr
+			return false
+		}
+	}
+	return true
+}
+
+// Converts given path to one supported by a merged /usr config.
+// E.g., /bin/foo becomes /usr/bin/foo, /lib/bar becomes /usr/lib/bar
+// See: https://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge
+func MergeUsr(file string) string {
+
+	// Prepend /usr to supported paths
+	for _, prefix := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
+		if strings.HasPrefix(file, prefix) {
+			file = filepath.Join("/usr", file)
+			break
+		}
+	}
+
+	return file
+}
+
 // Converts a relative symlink target path (e.g. ../../lib/foo.so), that is
 // absolute path
 func RelativeSymlinkTargetToDir(symPath string, dir string) (string, error) {

internal/osutil/osutil_test.go

@@ -0,0 +1,49 @@
+// Copyright 2024 Clayton Craft <clayton@craftyguy.net>
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package osutil
+
+import (
+	"testing"
+)
+
+func TestMergeUsr(t *testing.T) {
+	subtests := []struct {
+		in       string
+		expected string
+	}{
+		{
+			in:       "/bin/foo",
+			expected: "/usr/bin/foo",
+		},
+		{
+			in:       "/sbin/foo",
+			expected: "/usr/sbin/foo",
+		},
+		{
+			in:       "/usr/sbin/foo",
+			expected: "/usr/sbin/foo",
+		},
+		{
+			in:       "/usr/bin/foo",
+			expected: "/usr/bin/foo",
+		},
+		{
+			in:       "/lib/foo.so",
+			expected: "/usr/lib/foo.so",
+		},
+		{
+			in:       "/lib64/foo.so",
+			expected: "/usr/lib64/foo.so",
+		},
+	}
+
+	for _, st := range subtests {
+		t.Run(st.in, func(t *testing.T) {
+			out := MergeUsr(st.in)
+			if out != st.expected {
+				t.Fatalf("expected: %q, got: %q\n", st.expected, out)
+			}
+		})
+	}
+}

pkgs/deviceinfo/deviceinfo.go

@@ -4,87 +4,82 @@
 package deviceinfo
 
 import (
-	"bufio"
+	"context"
 	"fmt"
-	"io"
-	"log"
-	"os"
 	"reflect"
+	"strconv"
 	"strings"
+	"time"
+
+	"github.com/mvdan/sh/shell"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type DeviceInfo struct {
 	InitfsCompression      string
 	InitfsExtraCompression string
-	MesaDriver             string
-	ModulesInitfs          string
 	UbootBoardname         string
+	GenerateSystemdBoot    string
+	FormatVersion          string
+	CreateInitfsExtra      bool
 }
 
-func ReadDeviceinfo(file string) (DeviceInfo, error) {
+// Reads the relevant entries from "file" into DeviceInfo struct
-	var deviceinfo DeviceInfo
+// Any already-set entries will be overwriten if they are present
-
+// in "file"
-	fd, err := os.Open(file)
+func (d *DeviceInfo) ReadDeviceinfo(file string) error {
-	if err != nil {
+	if exists, err := misc.Exists(file); !exists {
-		return deviceinfo, err
+		return fmt.Errorf("%q not found, required by mkinitfs", file)
-	}
+	} else if err != nil {
-	defer fd.Close()
+		return fmt.Errorf("unexpected error getting status for %q: %s", file, err)
-
-	if err := unmarshal(fd, &deviceinfo); err != nil {
-		return deviceinfo, err
 	}
 
-	return deviceinfo, nil
+	if err := d.unmarshal(file); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Unmarshals a deviceinfo into a DeviceInfo struct
-func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
+func (d *DeviceInfo) unmarshal(file string) error {
-	s := bufio.NewScanner(r)
+	ctx, cancelCtx := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
-	for s.Scan() {
+	defer cancelCtx()
-		line := s.Text()
+	vars, err := shell.SourceFile(ctx, file)
-		if strings.HasPrefix(line, "#") {
+	if err != nil {
-			continue
+		return fmt.Errorf("parsing deviceinfo %q failed: %w", file, err)
-		}
+	}
 
-		// line isn't setting anything, so just ignore it
+	for k, v := range vars {
-		if !strings.Contains(line, "=") {
+		fieldName := nameToField(k)
-			continue
+		field := reflect.ValueOf(d).Elem().FieldByName(fieldName)
-		}
-
-		// sometimes line has a comment at the end after setting an option
-		line = strings.SplitN(line, "#", 2)[0]
-		line = strings.TrimSpace(line)
-
-		// must support having '=' in the value (e.g. kernel cmdline)
-		parts := strings.SplitN(line, "=", 2)
-		if len(parts) != 2 {
-			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
-		}
-
-		name, val := parts[0], parts[1]
-		val = strings.ReplaceAll(val, "\"", "")
-
-		if name == "deviceinfo_format_version" && val != "0" {
-			return fmt.Errorf("deviceinfo format version %q is not supported", val)
-		}
-
-		fieldName := nameToField(name)
-
-		if fieldName == "" {
-			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
-		}
-
-		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
 		if !field.IsValid() {
 			// an option that meets the deviceinfo "specification", but isn't
 			// one we care about in this module
 			continue
 		}
-		field.SetString(val)
+		switch field.Interface().(type) {
+		case string:
+			field.SetString(v.String())
+		case bool:
+			if v, err := strconv.ParseBool(v.String()); err != nil {
+				return fmt.Errorf("deviceinfo %q has unsupported type for field %q, expected 'bool'", file, k)
+			} else {
+				field.SetBool(v)
+			}
+		case int:
+			if v, err := strconv.ParseInt(v.String(), 10, 32); err != nil {
+				return fmt.Errorf("deviceinfo %q has unsupported type for field %q, expected 'int'", file, k)
+			} else {
+				field.SetInt(v)
+			}
+		default:
+			return fmt.Errorf("deviceinfo %q has unsupported type for field %q", file, k)
+		}
 	}
-	if err := s.Err(); err != nil {
+
-		log.Print("unable to parse deviceinfo: ", err)
+	if d.FormatVersion != "0" {
-		return err
+		return fmt.Errorf("deviceinfo %q has an unsupported format version %q", file, d.FormatVersion)
 	}
 
 	return nil
@@ -108,3 +103,25 @@ func nameToField(name string) string {
 
 	return field
 }
+
+func (d DeviceInfo) String() string {
+	return fmt.Sprintf(`{
+			%s: %v
+			%s: %v
+			%s: %v
+			%s: %v
+			%s: %v
+			%s: %v
+			%s: %v
+			%s: %v
+	}`,
+		"deviceinfo_format_version", d.FormatVersion,
+		"deviceinfo_", d.FormatVersion,
+		"deviceinfo_initfs_compression", d.InitfsCompression,
+		"deviceinfo_initfs_extra_compression", d.InitfsCompression,
+		"deviceinfo_ubootBoardname", d.UbootBoardname,
+		"deviceinfo_generateSystemdBoot", d.GenerateSystemdBoot,
+		"deviceinfo_formatVersion", d.FormatVersion,
+		"deviceinfo_createInitfsExtra", d.CreateInitfsExtra,
+	)
+}

pkgs/deviceinfo/deviceinfo_test.go

@@ -4,12 +4,32 @@
 package deviceinfo
 
 import (
-	"fmt"
-	"reflect"
 	"strings"
 	"testing"
 )
 
+// Test ReadDeviceinfo and the logic of reading from multiple files
+func TestReadDeviceinfo(t *testing.T) {
+	compression_expected := "gz -9"
+
+	var devinfo DeviceInfo
+	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
+	if !strings.Contains(err.Error(), "required by mkinitfs") {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-first")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-msm")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	if devinfo.InitfsCompression != compression_expected {
+		t.Errorf("expected %q, got: %q", compression_expected, devinfo.InitfsCompression)
+	}
+}
+
 // Test conversion of name to DeviceInfo struct field format
 func TestNameToField(t *testing.T) {
 	tables := []struct {
@@ -18,10 +38,11 @@ func TestNameToField(t *testing.T) {
 	}{
 		{"deviceinfo_dtb", "Dtb"},
 		{"dtb", "Dtb"},
-		{"deviceinfo_modules_initfs", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression", "InitfsCompression"},
 		{"modules_initfs", "ModulesInitfs"},
-		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression___", "InitfsCompression"},
 		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
+		{"deviceinfo_create_initfs_extra", "CreateInitfsExtra"},
 	}
 
 	for _, table := range tables {
@@ -37,39 +58,25 @@ func TestUnmarshal(t *testing.T) {
 	tables := []struct {
 		// field is just used for reflection within the test, so it must be a
 		// valid DeviceInfo field
-		field    string
+		file     string
-		in       string
+		expected DeviceInfo
-		expected string
 	}{
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
+		{"./test_resources/deviceinfo-unmarshal-1", DeviceInfo{
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
+			FormatVersion:          "0",
-		// line with multiple '='
+			UbootBoardname:         "foobar-bazz",
-		{"InitfsCompression", "deviceinfo_initfs_compression=zstd:--foo=1 -T0 --bar=bazz", "zstd:--foo=1 -T0 --bar=bazz"},
+			InitfsCompression:      "zstd:--foo=1 -T0 --bar=bazz",
-		// empty option
+			InitfsExtraCompression: "",
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
+			CreateInitfsExtra:      true,
-		// line with comment at the end
+		},
-		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
+		},
-		{"", "# this is a comment!\n", ""},
-		// empty lines are fine
-		{"", "", ""},
-		// line with whitepace characters only
-		{"", " \t \n\r", ""},
 	}
 	var d DeviceInfo
 	for _, table := range tables {
-		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
+		if err := d.unmarshal(table.file); err != nil {
-		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
+			t.Error(err)
-			t.Errorf("%s received an unexpected err: ", err)
 		}
-
+		if d != table.expected {
-		// Check against expected value
+			t.Errorf("expected: %s, got: %s", table.expected, d)
-		field := reflect.ValueOf(&d).Elem().FieldByName(table.field)
-		out := ""
-		if table.field != "" {
-			out = field.String()
-		}
-		if out != table.expected {
-			t.Errorf("%s expected: %q, got: %q", testName, table.expected, out)
 		}
 	}
 

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -0,0 +1,3 @@
+deviceinfo_format_version="0"
+deviceinfo_initfs_compression="gz -9"
+deviceinfo_mesa_driver="panfrost"

pkgs/deviceinfo/test_resources/deviceinfo-msm

@@ -0,0 +1,2 @@
+deviceinfo_format_version="0"
+deviceinfo_mesa_driver="msm"

pkgs/deviceinfo/test_resources/deviceinfo-unmarshal-1

@@ -0,0 +1,7 @@
+deviceinfo_format_version="0"
+deviceinfo_uboot_boardname="foobar-bazz"
+# line with multiple =
+deviceinfo_initfs_compression="zstd:--foo=1 -T0 --bar=bazz"
+# empty option
+deviceinfo_initfs_extra_compression=""
+deviceinfo_create_initfs_extra="true" # in-line comment that should be ignored