misc: also check wheter binaries from /{s,}bin are /usr

This should hopefully avoid failing to build the initramfs
when important binaries like /bin/busybox get moved to /usr.

Also complain loudly when there's a path mismatch, so people notice
it.

cmd/mkinitfs/main.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/archive"
@@ -20,7 +19,6 @@ import (
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/hookscripts"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/initramfs"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
@@ -105,8 +103,8 @@ func main() {
 		hookfiles.New("/etc/mkinitfs/files"),
 		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
 		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
-		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
+		modules.New("/usr/share/mkinitfs/modules"),
-		modules.New([]string{}, "/etc/mkinitfs/modules"),
+		modules.New("/etc/mkinitfs/modules"),
 	})
 	if err := initramfsAr.AddItems(initfs); err != nil {
 		log.Println(err)
@@ -137,9 +135,8 @@ func main() {
 		hookfiles.New("/etc/mkinitfs/files-extra"),
 		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
 		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
-		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
+		modules.New("/usr/share/mkinitfs/modules-extra"),
-		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
+		modules.New("/etc/mkinitfs/modules-extra"),
-		osksdl.New(devinfo.MesaDriver),
 	})
 	if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
 		log.Println(err)
@@ -157,7 +154,7 @@ func main() {
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
-		if err := bootDeploy(workDir, *outDir, devinfo.UbootBoardname); err != nil {
+		if err := bootDeploy(workDir, *outDir, devinfo); err != nil {
 			log.Println(err)
 			log.Println("boot-deploy failed")
 			retCode = 1
@@ -166,10 +163,10 @@ func main() {
 	}
 }
 
-func bootDeploy(workDir, outDir, ubootBoardname string) error {
+func bootDeploy(workDir string, outDir string, devinfo deviceinfo.DeviceInfo) error {
 	log.Print("== Using boot-deploy to finalize/install files ==")
 	defer misc.TimeFunc(time.Now(), "boot-deploy")
 
-	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
+	bd := bootdeploy.New(workDir, outDir, devinfo)
 	return bd.Run()
 }

doc/mkinitfs.1.scd

@@ -42,10 +42,9 @@ mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
 */etc/deviceinfo*, in that order. The following variables
 are *required* by mkinitfs:
 
+	- deviceinfo_generate_systemd_boot
 	- deviceinfo_initfs_compression
 	- deviceinfo_initfs_extra_compression
-	- deviceinfo_mesa_driver
-	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
 
 It is a design goal to keep the number of required variables from deviceinfo to
@@ -55,6 +54,36 @@ a bare minimum, and to require only variables that don't hold lists of things.
 necessary tools to extract the configured archive format are in the initramfs
 archive.
 
+# ARCHIVE COMPRESSION
+
+Archive compression parameters are specified in the
+*deviceinfo_initfs_compression* and *deviceinfo_initfs_extra_compression*
+deviceinfo variables. Their values do not have to match, but special
+consideration should be taken since some formats may require additional kernel
+options or tools in the initramfs to support it.
+
+Supported compression *formats* for mkinitfs are:
+
+	- gzip
+	- lz4
+	- lzma
+	- none
+	- zstd
+
+Supported compression *levels* for mkinitfs:
+
+	- best
+	- default
+	- fast
+
+The value of these variables follows this syntax: *<format>:<level>*. For
+example, *zstd* with the *fast* compression level would be:
+*deviceinfo_initfs_compression="zstd:fast"*
+
+Defaults to *gzip* and *default* for both archives if format and/or level is
+unsupported or omitted.
+
+
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and

internal/bootdeploy/bootdeploy.go

@@ -10,32 +10,32 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
 
 type BootDeploy struct {
 	inDir   string
 	outDir  string
-	ubootBoardname string
+	devinfo deviceinfo.DeviceInfo
 }
 
 // New returns a new BootDeploy, which then runs:
 //
 //	boot-deploy -d indir -o outDir
 //
-// ubootBoardname is used for copying in some u-boot files prior to running
+// devinfo is used to access some deviceinfo values, such as UbootBoardname
-// boot-deploy. This is optional, passing an empty string is ok if this is not
+// and GenerateSystemdBoot
-// needed.
+func New(inDir string, outDir string, devinfo deviceinfo.DeviceInfo) *BootDeploy {
-func New(inDir, outDir, ubootBoardname string) *BootDeploy {
 	return &BootDeploy{
 		inDir:   inDir,
 		outDir:  outDir,
-		ubootBoardname: ubootBoardname,
+		devinfo: devinfo,
 	}
 }
 
 func (b *BootDeploy) Run() error {
-
+	if err := copyUbootFiles(b.inDir, b.devinfo.UbootBoardname); errors.Is(err, os.ErrNotExist) {
-	if err := copyUbootFiles(b.inDir, b.ubootBoardname); errors.Is(err, os.ErrNotExist) {
 		log.Println("u-boot files copying skipped: ", err)
 	} else {
 		if err != nil {
@@ -43,15 +43,9 @@ func (b *BootDeploy) Run() error {
 		}
 	}
 
-	return bootDeploy(b.inDir, b.outDir)
+	kernels, err := getKernelPath(b.outDir, b.devinfo.GenerateSystemdBoot == "true")
-}
+	if err != nil {
-
+		return err
-func bootDeploy(workDir string, outDir string) error {
-	// boot-deploy expects the kernel to be in the same dir as initramfs.
-	// Assume that the kernel is in the output dir...
-	kernels, _ := filepath.Glob(filepath.Join(outDir, "vmlinuz*"))
-	if len(kernels) == 0 {
-		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
 	}
 
 	// Pick a kernel that does not have suffixes added by boot-deploy
@@ -71,7 +65,7 @@ func bootDeploy(workDir string, outDir string) error {
 	defer kernFd.Close()
 
 	kernFilename := path.Base(kernFile)
-	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
+	kernFileCopy, err := os.Create(filepath.Join(b.inDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -87,8 +81,8 @@ func bootDeploy(workDir string, outDir string) error {
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
 		"-k", kernFilename,
-		"-d", workDir,
+		"-d", b.inDir,
-		"-o", outDir,
+		"-o", b.outDir,
 		"initramfs-extra")
 
 	cmd.Stdout = os.Stdout
@@ -100,6 +94,25 @@ func bootDeploy(workDir string, outDir string) error {
 	return nil
 }
 
+func getKernelPath(outDir string, zboot bool) ([]string, error) {
+	var kernels []string
+	if zboot {
+		kernels, _ = filepath.Glob(filepath.Join(outDir, "linux.efi"))
+		if len(kernels) > 0 {
+			return kernels, nil
+		}
+		// else fallback to vmlinuz* below
+	}
+
+	kernFile := "vmlinuz*"
+	kernels, _ = filepath.Glob(filepath.Join(outDir, kernFile))
+	if len(kernels) == 0 {
+		return nil, errors.New("Unable to find any kernels at " + filepath.Join(outDir, kernFile))
+	}
+
+	return kernels, nil
+}
+
 // Copy copies the file at srcFile path to a new file at dstFile path
 func copy(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)

internal/filelist/modules/modules.go

@@ -17,14 +17,11 @@ import (
 
 type Modules struct {
 	modulesListPath string
-	modulesList     []string
 }
 
-// New returns a new Modules that will use the given moduleto provide a list
+// New returns a new Modules that will read in lists of kernel modules in the given path.
-// of script files.
+func New(modulesListPath string) *Modules {
-func New(modulesList []string, modulesListPath string) *Modules {
 	return &Modules{
-		modulesList:     modulesList,
 		modulesListPath: modulesListPath,
 	}
 }
@@ -52,20 +49,6 @@ func (m *Modules) List() (*filelist.FileList, error) {
 		files.Add(file, file)
 	}
 
-	// slurp up given list of modules
-	if len(m.modulesList) > 0 {
-		log.Printf("-- Including kernel modules from deviceinfo")
-		for _, module := range m.modulesList {
-			if modFilelist, err := getModule(module, modDir); err != nil {
-				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
-			} else {
-				for _, file := range modFilelist {
-					files.Add(file, file)
-				}
-			}
-		}
-	}
-
 	// slurp up modules from lists in modulesListPath
 	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
@@ -135,7 +118,9 @@ func getModulesInDir(modPath string) (files []string, err error) {
 			// Unable to walk path
 			return err
 		}
-		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
+		// this assumes module names are in the format <name>.ko[.format],
+		// where ".format" (e.g. ".gz") is optional.
+		if !strings.Contains(".ko", path) {
 			return nil
 		}
 		files = append(files, path)

internal/filelist/osksdl/osksdl.go

@@ -1,158 +0,0 @@
-package osksdl
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
-)
-
-type OskSdl struct {
-	mesaDriver string
-}
-
-// New returns a new HookScripts that will use the given path to provide a list
-// of script files.
-func New(mesaDriverName string) *OskSdl {
-	return &OskSdl{
-		mesaDriver: mesaDriverName,
-	}
-}
-
-// Get a list of files and their dependencies related to supporting rootfs full
-// disk (d)encryption
-func (s *OskSdl) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
-
-	if exists, err := misc.Exists("/usr/bin/osk-sdl"); !exists {
-		return files, nil
-	} else if err != nil {
-		return files, fmt.Errorf("received unexpected error when getting status for %q: %w", "/usr/bin/osk-sdl", err)
-	}
-
-	log.Println("- Including osk-sdl support")
-
-	confFiles := []string{
-		"/etc/osk.conf",
-		"/etc/ts.conf",
-		"/etc/pointercal",
-		"/etc/fb.modes",
-		"/etc/directfbrc",
-	}
-	confFileList, err := misc.GetFiles(confFiles, false)
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	}
-	for _, file := range confFileList {
-		files.Add(file, file)
-	}
-
-	// osk-sdl
-	oskFiles := []string{
-		"/usr/bin/osk-sdl",
-		"/sbin/cryptsetup",
-		"/usr/lib/libGL.so.1",
-	}
-	if oskFileList, err := misc.GetFiles(oskFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range oskFileList {
-			files.Add(file, file)
-		}
-	}
-
-	fontFile, err := getOskConfFontPath("/etc/osk.conf")
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
-	}
-	files.Add(fontFile, fontFile)
-
-	// Directfb
-	dfbFiles := []string{}
-	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			dfbFiles = append(dfbFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
-	}
-	if dfbFileList, err := misc.GetFiles(dfbFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range dfbFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// tslib
-	tslibFiles := []string{}
-	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
-		if filepath.Ext(path) == ".so" {
-			tslibFiles = append(tslibFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
-	}
-	libts, _ := filepath.Glob("/usr/lib/libts*")
-	tslibFiles = append(tslibFiles, libts...)
-	if tslibFileList, err := misc.GetFiles(tslibFiles, true); err != nil {
-		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	} else {
-		for _, file := range tslibFileList {
-			files.Add(file, file)
-		}
-	}
-
-	// mesa hw accel
-	if s.mesaDriver != "" {
-		mesaFiles := []string{
-			"/usr/lib/libEGL.so.1",
-			"/usr/lib/libGLESv2.so.2",
-			"/usr/lib/libgbm.so.1",
-			"/usr/lib/libudev.so.1",
-			"/usr/lib/xorg/modules/dri/" + s.mesaDriver + "_dri.so",
-		}
-		if mesaFileList, err := misc.GetFiles(mesaFiles, true); err != nil {
-			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-		} else {
-			for _, file := range mesaFileList {
-				files.Add(file, file)
-			}
-		}
-	}
-
-	return files, nil
-}
-
-func getOskConfFontPath(oskConfPath string) (string, error) {
-	var path string
-	f, err := os.Open(oskConfPath)
-	if err != nil {
-		return path, err
-	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		fields := strings.Fields(s.Text())
-		// "key = val" is 3 fields
-		if len(fields) > 2 && fields[0] == "keyboard-font" {
-			path = fields[2]
-		}
-	}
-	if exists, err := misc.Exists(path); !exists {
-		return path, fmt.Errorf("unable to find font: %s", path)
-	} else if err != nil {
-		return path, fmt.Errorf("received unexpected error when getting status for %q: %w", path, err)
-	}
-
-	return path, nil
-}

internal/misc/getfiles.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
+	"log"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
 )
@@ -22,30 +24,54 @@ func GetFiles(list []string, required bool) (files []string, err error) {
 	return
 }
 
-func getFile(file string, required bool) (files []string, err error) {
+// This function doesn't handle globs, use getFile() instead.
-	// Expand glob expression
+func getFileNormalized(file string, required bool) (files []string, err error) {
-	expanded, err := filepath.Glob(file)
+	fileInfo, err := os.Stat(file)
+
+	// Trying some fallbacks...
 	if err != nil {
-		return
+		type triedResult struct {
-	}
+			file string
-	if len(expanded) > 0 && expanded[0] != file {
+			err error
-		for _, path := range expanded {
-			if globFiles, err := getFile(path, required); err != nil {
-				return files, err
-			} else {
-				files = append(files, globFiles...)
-			}
-		}
-		return RemoveDuplicates(files), nil
 		}
 
-	fileInfo, err := os.Stat(file)
+		triedFiles := make([]triedResult, 0, 1)
-	if err != nil {
+
-		if required {
+		// Temporary fallback until alpine/pmOS usr-merge happened
-			return files, fmt.Errorf("getFile: failed to stat file %q: %w", file, err)
+		// If a path starts with /bin or /sbin, also try /usr equivalent before giving up
+		if strings.HasPrefix(file, "/bin/") || strings.HasPrefix(file, "/sbin/") {
+			fileUsr := filepath.Join("/usr", file)
+			_, err := os.Stat(fileUsr);
+			if err == nil {
+				log.Printf("getFile: failed to find %q, but found it in %q. Please adjust the path.", file, fileUsr)
+				return getFileNormalized(fileUsr, required)
+			} else {
+				triedFiles = append(triedFiles, triedResult{fileUsr, err})
 			}
+		}
+
+		{
+			// Check if there is a Zstd-compressed version of the file
+			fileZstd := file + ".zst" // .zst is the extension used by linux-firmware
+			_, err := os.Stat(fileZstd);
+			if err == nil {
+				return getFileNormalized(fileZstd, required)
+			} else {
+				triedFiles = append(triedFiles, triedResult{fileZstd, err})
+			}
+		}
+		
+		// Failed to find anything
+		if required {
+			failStrings := make([]string, 0, 2)
+			for _, result := range triedFiles {
+				failStrings = append(failStrings, fmt.Sprintf("\n - also tried %q: %v", result.file, result.err))
+			}
+			return files, fmt.Errorf("getFile: failed to stat file %q: %v%q", file, err, strings.Join(failStrings, ""))
+		} else {
 			return files, nil
 		}
+	}
 
 	if fileInfo.IsDir() {
 		// Recurse over directory contents
@@ -83,6 +109,26 @@ func getFile(file string, required bool) (files []string, err error) {
 	return
 }
 
+func getFile(file string, required bool) (files []string, err error) {
+	// Expand glob expression
+	expanded, err := filepath.Glob(file)
+	if err != nil {
+		return
+	}
+	if len(expanded) > 0 && expanded[0] != file {
+		for _, path := range expanded {
+			if globFiles, err := getFile(path, required); err != nil {
+				return files, err
+			} else {
+				files = append(files, globFiles...)
+			}
+		}
+		return RemoveDuplicates(files), nil
+	}
+
+	return getFileNormalized(file, required)
+}
+
 func getDeps(file string, parents map[string]struct{}) (files []string, err error) {
 
 	if _, found := parents[file]; found {

pkgs/deviceinfo/deviceinfo.go

@@ -18,9 +18,8 @@ import (
 type DeviceInfo struct {
 	InitfsCompression      string
 	InitfsExtraCompression string
-	MesaDriver             string
-	ModulesInitfs          string
 	UbootBoardname         string
+	GenerateSystemdBoot    string
 }
 
 // Reads the relevant entries from "file" into DeviceInfo struct

pkgs/deviceinfo/deviceinfo_test.go

@@ -12,8 +12,7 @@ import (
 
 // Test ReadDeviceinfo and the logic of reading from multiple files
 func TestReadDeviceinfo(t *testing.T) {
-	modules_expected := "panfrost foo bar bazz"
+	compression_expected := "gz -9"
-	mesa_expected := "msm"
 
 	var devinfo DeviceInfo
 	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
@@ -28,11 +27,8 @@ func TestReadDeviceinfo(t *testing.T) {
 	if err != nil {
 		t.Errorf("received an unexpected err: %s", err)
 	}
-	if devinfo.ModulesInitfs != modules_expected {
+	if devinfo.InitfsCompression != compression_expected {
-		t.Errorf("expected %q, got: %q", modules_expected, devinfo.ModulesInitfs)
+		t.Errorf("expected %q, got: %q", compression_expected, devinfo.InitfsCompression)
-	}
-	if devinfo.MesaDriver != mesa_expected {
-		t.Errorf("expected %q, got: %q", mesa_expected, devinfo.MesaDriver)
 	}
 }
 
@@ -44,9 +40,9 @@ func TestNameToField(t *testing.T) {
 	}{
 		{"deviceinfo_dtb", "Dtb"},
 		{"dtb", "Dtb"},
-		{"deviceinfo_modules_initfs", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression", "InitfsCompression"},
 		{"modules_initfs", "ModulesInitfs"},
-		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_compression___", "InitfsCompression"},
 		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
@@ -67,14 +63,12 @@ func TestUnmarshal(t *testing.T) {
 		in       string
 		expected string
 	}{
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"gzip:-9\"\n", "gzip:-9"},
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
 		// line with multiple '='
 		{"InitfsCompression", "deviceinfo_initfs_compression=zstd:--foo=1 -T0 --bar=bazz", "zstd:--foo=1 -T0 --bar=bazz"},
 		// empty option
-		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
+		{"InitfsCompression", "deviceinfo_initfs_compression=\"\"\n", ""},
 		// line with comment at the end
-		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
 		{"", "# this is a comment!\n", ""},
 		// empty lines are fine
 		{"", "", ""},

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -1,2 +1,2 @@
-deviceinfo_modules_initfs="panfrost foo bar bazz"
+deviceinfo_initfs_compression="gz -9"
 deviceinfo_mesa_driver="panfrost"