filelist/osk-sdl: add deprecation warning

Co-authored-by: Oliver Smith <ollieparanoid@postmarketos.org>

cmd/mkinitfs/main.go

@@ -47,26 +47,19 @@ func main() {
 		return
 	}
 
-	deviceinfoFile := "/etc/deviceinfo"
+	log.Default().SetFlags(log.Lmicroseconds)
-	if exists, err := misc.Exists(deviceinfoFile); !exists {
-		log.Printf("NOTE: %q not found, this file is required by mkinitfs.\n", deviceinfoFile)
-		return
-	} else if err != nil {
-		retCode = 1
-		log.Printf("received unexpected error when getting status for %q: %s", deviceinfoFile, err)
-	}
 
-	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
+	var devinfo deviceinfo.DeviceInfo
-	if err != nil {
+	deverr_usr := devinfo.ReadDeviceinfo("/usr/share/deviceinfo/deviceinfo")
-		log.Println(err)
+	deverr_etc := devinfo.ReadDeviceinfo("/etc/deviceinfo")
+	if deverr_etc != nil && deverr_usr != nil {
+		log.Println("Error reading deviceinfo")
+		log.Println("\t/usr/share/deviceinfo/deviceinfo:", deverr_usr)
+		log.Println("\t/etc/deviceinfo:", deverr_etc)
 		retCode = 1
 		return
 	}
 
-	// deviceinfo.InitfsCompression needs a little more post-processing
-	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
-	log.Printf("Using compression format %s with level %q\n", compressionFormat, compressionLevel)
-
 	defer misc.TimeFunc(time.Now(), "mkinitfs")
 
 	kernVer, err := osutil.GetKernelVersion()
@@ -87,47 +80,80 @@ func main() {
 	defer func() {
 		e := os.RemoveAll(workDir)
 		if e != nil && err == nil {
-			err = e
+			log.Println(e)
-			retCode = 1
+			log.Println("unable to remove temporary work directory")
 		}
 	}()
 
 	log.Print("Generating for kernel version: ", kernVer)
 	log.Print("Output directory: ", *outDir)
 
-	if err := generateArchive("initramfs", compressionFormat, compressionLevel, workDir, []filelist.FileLister{
+	//
+	// initramfs
+	//
+	// deviceinfo.InitfsCompression needs a little more post-processing
+	compressionFormat, compressionLevel := archive.ExtractFormatLevel(devinfo.InitfsCompression)
+	log.Printf("== Generating %s ==\n", "initramfs")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start := time.Now()
+	initramfsAr := archive.New(compressionFormat, compressionLevel)
+	initfs := initramfs.New([]filelist.FileLister{
 		hookdirs.New("/usr/share/mkinitfs/dirs"),
 		hookdirs.New("/etc/mkinitfs/dirs"),
 		hookfiles.New("/usr/share/mkinitfs/files"),
 		hookfiles.New("/etc/mkinitfs/files"),
-		hookscripts.New("/usr/share/mkinitfs/hooks"),
+		hookscripts.New("/usr/share/mkinitfs/hooks", "/hooks"),
-		hookscripts.New("/etc/mkinitfs/hooks"),
+		hookscripts.New("/etc/mkinitfs/hooks", "/hooks"),
 		modules.New(strings.Fields(devinfo.ModulesInitfs), "/usr/share/mkinitfs/modules"),
 		modules.New([]string{}, "/etc/mkinitfs/modules"),
-	}); err != nil {
+	})
+	if err := initramfsAr.AddItems(initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs")
 		retCode = 1
 		return
 	}
+	if err := initramfsAr.Write(filepath.Join(workDir, "initramfs"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs")
 
-	// Note: compression disabled for initramfs-extra, since it slows down boot
+	//
-	// and can add more requirements to the initramfs (e.g. need to add support
+	// initramfs-extra
-	// for extracting zstd)
+	//
-	if err := generateArchive("initramfs-extra", archive.FormatNone, archive.LevelDefault, workDir, []filelist.FileLister{
+	// deviceinfo.InitfsExtraCompression needs a little more post-processing
+	compressionFormat, compressionLevel = archive.ExtractFormatLevel(devinfo.InitfsExtraCompression)
+	log.Printf("== Generating %s ==\n", "initramfs-extra")
+	log.Printf("- Using compression format %s with level %q\n", compressionFormat, compressionLevel)
+
+	start = time.Now()
+	initramfsExtraAr := archive.New(compressionFormat, compressionLevel)
+	initfsExtra := initramfs.New([]filelist.FileLister{
 		hookfiles.New("/usr/share/mkinitfs/files-extra"),
 		hookfiles.New("/etc/mkinitfs/files-extra"),
-		hookscripts.New("/usr/share/mkinitfs/hooks-extra"),
+		hookscripts.New("/usr/share/mkinitfs/hooks-extra", "/hooks-extra"),
-		hookscripts.New("/etc/mkinitfs/hooks-extra"),
+		hookscripts.New("/etc/mkinitfs/hooks-extra", "/hooks-extra"),
 		modules.New([]string{}, "/usr/share/mkinitfs/modules-extra"),
 		modules.New([]string{}, "/etc/mkinitfs/modules-extra"),
 		osksdl.New(devinfo.MesaDriver),
-	}); err != nil {
+	})
+	if err := initramfsExtraAr.AddItemsExclude(initfsExtra, initfs); err != nil {
 		log.Println(err)
 		log.Println("failed to generate: ", "initramfs-extra")
 		retCode = 1
 		return
 	}
+	if err := initramfsExtraAr.Write(filepath.Join(workDir, "initramfs-extra"), os.FileMode(0644)); err != nil {
+		log.Println(err)
+		log.Println("failed to generate: ", "initramfs-extra")
+		retCode = 1
+		return
+	}
+	misc.TimeFunc(start, "initramfs-extra")
 
 	// Final processing of initramfs / kernel is done by boot-deploy
 	if !disableBootDeploy {
@@ -147,24 +173,3 @@ func bootDeploy(workDir, outDir, ubootBoardname string) error {
 	bd := bootdeploy.New(workDir, outDir, ubootBoardname)
 	return bd.Run()
 }
-
-func generateArchive(name string, format archive.CompressFormat, level archive.CompressLevel, path string, features []filelist.FileLister) error {
-	log.Printf("== Generating %s ==\n", name)
-	defer misc.TimeFunc(time.Now(), name)
-	a, err := archive.New(format, level)
-	if err != nil {
-		return err
-	}
-
-	fs := initramfs.New(features)
-	if err := a.AddItems(fs); err != nil {
-		return err
-	}
-
-	log.Println("- Writing and verifying archive: ", name)
-	if err := a.Write(filepath.Join(path, name), os.FileMode(0644)); err != nil {
-		return err
-	}
-
-	return nil
-}

doc/mkinitfs.1.scd

@@ -24,15 +24,26 @@ is purely to generate the archive(s). mkinitfs does call *boot-deploy* after
 creating the archive(s), in order to install/deploy them and any other relevant
 boot-related items onto the system.
 
+Design goals of this project are:
+
+	- Support as many distros as possible
+	- Simplify configuration, while still giving multiple opportunities to set or override defaults
+	- Execute an external app to do any boot install/setup finalization
+		- One such app is here: https://gitlab.com/postmarketOS/boot-deploy
+		- But implementation can be anything, see the section on *BOOT-DEPLOY*
+		for more info
+
 # DEVICEINFO
 
 The canonical deviceinfo "specification" is at
 https://wiki.postmarketos.org/wiki/Deviceinfo_reference
 
-mkinitfs reads deviceinfo values from */etc/deviceinfo*. The following variables
+mkinitfs reads deviceinfo values from */usr/share/deviceinfo/deviceinfo* and
+*/etc/deviceinfo*, in that order. The following variables
 are *required* by mkinitfs:
 
 	- deviceinfo_initfs_compression
+	- deviceinfo_initfs_extra_compression
 	- deviceinfo_mesa_driver
 	- deviceinfo_modules_initfs
 	- deviceinfo_uboot_boardname
@@ -40,6 +51,10 @@ are *required* by mkinitfs:
 It is a design goal to keep the number of required variables from deviceinfo to
 a bare minimum, and to require only variables that don't hold lists of things.
 
+*NOTE*: When deviceinfo_initfs_extra_compression is set, make sure that the
+necessary tools to extract the configured archive format are in the initramfs
+archive.
+
 # DIRECTORIES
 
 The following directories are used by mkinitfs to generate the initramfs and
@@ -49,7 +64,7 @@ it are for constructing the initramfs archive.
 
 Configuration under */usr/share/mkinitfs* is intended to be managed by
 distributions, while configuration under */etc/mkinitfs* is for users to
-create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*. 
+create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, and then from */etc/mkinitfs*.
 
 ## /usr/share/mkinitfs/files, /etc/mkinitfs/files
 ## /usr/share/mkinitfs/files-extra, /etc/mkinitfs/files-extra
@@ -85,6 +100,9 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	path(s) under the relevant directory in */etc/mkinitfs*, and changing
 	the destination path.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/hooks, /etc/mkinitfs/hooks
 ## /usr/share/mkinitfs/hooks-extra*, /etc/mkinitfs/hooks-extra
 
@@ -106,12 +124,42 @@ create/manage. mkinitfs reads configuration from */usr/share/mkinitfs* first, an
 	Modules are installed in the initramfs archive under the same path they
 	exist on the system where mkinitfs is executed.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
 ## /usr/share/mkinitfs/dirs, /etc/mkinitfs/dirs
 
 	Files with the *.dirs* extension in these directories are lists of
 	directories to create within the initramfs. There is no *-extra* variant,
 	since directories are of negligible size.
 
+	Any lines in these files that start with *#* are considered comments, and
+	skipped.
+
+# BOOT-DEPLOY
+
+After generating archives, mkinitfs will execute *boot-deploy*, using *$PATH* to
+search for the app. The following commandline options are passed to it:
+
+	*-i* <initramfs filename>
+
+		Currently this is hardcoded to be "initramfs"
+
+	*-k* <kernel filename>
+
+	*-d* <work directory>
+
+		Path to the directory containing the build artifacts from mkinitfs.
+
+	*-o* <destination directory>
+
+		Path to the directory that boot-deploy should use as its root when
+		installing files.
+
+	*initramfs-extra*
+
+		This string is the filename of the initramfs-extra archive.
+
 # AUTHORS
 
 *Clayton Craft* <clayton@craftyguy.net>

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.15.12
+	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/ulikunitz/xz v0.5.10
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 )

go.sum

@@ -2,6 +2,8 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RS
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
 github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
 github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=

internal/archive/archive.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/cavaliercoder/go-cpio"
 	"github.com/klauspost/compress/zstd"
+	"github.com/pierrec/lz4/v4"
 	"github.com/ulikunitz/xz"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/osutil"
@@ -28,6 +29,7 @@ type CompressFormat string
 const (
 	FormatGzip CompressFormat = "gzip"
 	FormatLzma CompressFormat = "lzma"
+	FormatLz4  CompressFormat = "lz4"
 	FormatZstd CompressFormat = "zstd"
 	FormatNone CompressFormat = "none"
 )
@@ -44,14 +46,14 @@ const (
 )
 
 type Archive struct {
-	items           archiveItems
 	cpioWriter      *cpio.Writer
 	buf             *bytes.Buffer
 	compress_format CompressFormat
 	compress_level  CompressLevel
+	items           archiveItems
 }
 
-func New(format CompressFormat, level CompressLevel) (*Archive, error) {
+func New(format CompressFormat, level CompressLevel) *Archive {
 	buf := new(bytes.Buffer)
 	archive := &Archive{
 		cpioWriter:      cpio.NewWriter(buf),
@@ -60,12 +62,12 @@ func New(format CompressFormat, level CompressLevel) (*Archive, error) {
 		compress_level:  level,
 	}
 
-	return archive, nil
+	return archive
 }
 
 type archiveItem struct {
-	sourcePath string
 	header     *cpio.Header
+	sourcePath string
 }
 
 type archiveItems struct {
@@ -104,6 +106,7 @@ func ExtractFormatLevel(s string) (format CompressFormat, level CompressLevel) {
 	case FormatLzma:
 		log.Println("Format lzma doesn't support a compression level, using default settings")
 		level = LevelDefault
+	case FormatLz4:
 	case FormatNone:
 	case FormatZstd:
 	default:
@@ -184,10 +187,11 @@ func (archive *Archive) Write(path string, mode os.FileMode) error {
 	return nil
 }
 
-// Adds the given items in the map to the archive. The map format is {source path:dest path}.
+// AddItems adds the given items in the map to the archive. The map format is
-// Internally this just calls AddItem on each key,value pair in the map.
+// {source path:dest path}. Internally this just calls AddItem on each
-func (archive *Archive) AddItems(f filelist.FileLister) error {
+// key,value pair in the map.
-	list, err := f.List()
+func (archive *Archive) AddItems(flister filelist.FileLister) error {
+	list, err := flister.List()
 	if err != nil {
 		return err
 	}
@@ -199,6 +203,38 @@ func (archive *Archive) AddItems(f filelist.FileLister) error {
 	return nil
 }
 
+// AddItemsExclude is like AddItems, but takes a second FileLister that lists
+// items that should not be added to the archive from the first FileLister
+func (archive *Archive) AddItemsExclude(flister filelist.FileLister, exclude filelist.FileLister) error {
+	list, err := flister.List()
+	if err != nil {
+		return err
+	}
+
+	excludeList, err := exclude.List()
+	if err != nil {
+		return err
+	}
+
+	for i := range list.IterItems() {
+		dest, found := excludeList.Get(i.Source)
+
+		if found {
+			if i.Dest != dest {
+				found = false
+			}
+		}
+
+		if !found {
+			if err := archive.AddItem(i.Source, i.Dest); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // Adds the given file or directory at "source" to the archive at "dest"
 func (archive *Archive) AddItem(source string, dest string) error {
 
@@ -315,6 +351,23 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) (err erro
 		if err != nil {
 			return err
 		}
+	case FormatLz4:
+		// The default compression for the lz4 library is Fast, and
+		// they don't define a Default level otherwise
+		level := lz4.Fast
+		switch archive.compress_level {
+		case LevelBest:
+			level = lz4.Level9
+		case LevelFast:
+			level = lz4.Fast
+		}
+
+		var writer = lz4.NewWriter(fd)
+		err = writer.Apply(lz4.LegacyOption(true), lz4.CompressionLevelOption(level))
+		if err != nil {
+			return err
+		}
+		compressor = writer
 	case FormatNone:
 		compressor = fd
 	case FormatZstd:

internal/archive/archive_test.go

@@ -249,6 +249,12 @@ func TestExtractFormatLevel(t *testing.T) {
 			expectedFormat: FormatLzma,
 			expectedLevel:  LevelDefault,
 		},
+		{
+			name:           "lz4, fast",
+			in:             "lz4:fast",
+			expectedFormat: FormatLz4,
+			expectedLevel:  LevelFast,
+		},
 		{
 			name:           "none",
 			in:             "none",

internal/bootdeploy/bootdeploy.go

@@ -2,10 +2,12 @@ package bootdeploy
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -68,7 +70,8 @@ func bootDeploy(workDir string, outDir string) error {
 	}
 	defer kernFd.Close()
 
-	kernFileCopy, err := os.Create(filepath.Join(workDir, "vmlinuz"))
+	kernFilename := path.Base(kernFile)
+	kernFileCopy, err := os.Create(filepath.Join(workDir, kernFilename))
 	if err != nil {
 		return err
 	}
@@ -76,12 +79,14 @@ func bootDeploy(workDir string, outDir string) error {
 	if _, err = io.Copy(kernFileCopy, kernFd); err != nil {
 		return err
 	}
-	kernFileCopy.Close()
+	if err := kernFileCopy.Close(); err != nil {
+		return fmt.Errorf("error closing %s: %w", kernFilename, err)
+	}
 
 	// boot-deploy -i initramfs -k vmlinuz-postmarketos-rockchip -d /tmp/cpio -o /tmp/foo initramfs-extra
 	cmd := exec.Command("boot-deploy",
 		"-i", "initramfs",
-		"-k", "vmlinuz",
+		"-k", kernFilename,
 		"-d", workDir,
 		"-o", outDir,
 		"initramfs-extra")

internal/filelist/hookdirs/hookdirs.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist"
 )
@@ -23,7 +24,7 @@ func New(path string) *HookDirs {
 }
 
 func (h *HookDirs) List() (*filelist.FileList, error) {
-	log.Printf("- Creating directories specified in %s", h.path)
+	log.Printf("- Searching for directories specified in %s", h.path)
 
 	files := filelist.NewFileList()
 	fileInfo, err := os.ReadDir(h.path)
@@ -44,6 +45,10 @@ func (h *HookDirs) List() (*filelist.FileList, error) {
 		s := bufio.NewScanner(f)
 		for s.Scan() {
 			dir := s.Text()
+			if len(dir) == 0 || strings.HasPrefix(dir, "#") {
+				continue
+			}
+
 			files.Add(dir, dir)
 		}
 	}

internal/filelist/hookfiles/hookfiles.go

@@ -26,7 +26,7 @@ func New(filePath string) *HookFiles {
 }
 
 func (h *HookFiles) List() (*filelist.FileList, error) {
-	log.Printf("- Including file lists from %s", h.filePath)
+	log.Printf("- Searching for file lists from %s", h.filePath)
 
 	files := filelist.NewFileList()
 	fileInfo, err := os.ReadDir(h.filePath)
@@ -58,7 +58,12 @@ func slurpFiles(fd io.Reader) (*filelist.FileList, error) {
 
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
-		src, dest, has_dest := strings.Cut(s.Text(), ":")
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		src, dest, has_dest := strings.Cut(line, ":")
 
 		fFiles, err := misc.GetFiles([]string{src}, true)
 		if err != nil {

internal/filelist/hookscripts/hookscripts.go

@@ -9,19 +9,22 @@ import (
 )
 
 type HookScripts struct {
+	destPath   string
 	scriptsDir string
 }
 
 // New returns a new HookScripts that will use the given path to provide a list
-// of script files.
+// of script files. The destination for each script it set to destPath, using
-func New(scriptsDir string) *HookScripts {
+// the original file name.
+func New(scriptsDir string, destPath string) *HookScripts {
 	return &HookScripts{
+		destPath:   destPath,
 		scriptsDir: scriptsDir,
 	}
 }
 
 func (h *HookScripts) List() (*filelist.FileList, error) {
-	log.Printf("- Including hook scripts from %s", h.scriptsDir)
+	log.Printf("- Searching for hook scripts from %s", h.scriptsDir)
 
 	files := filelist.NewFileList()
 
@@ -32,7 +35,8 @@ func (h *HookScripts) List() (*filelist.FileList, error) {
 	}
 	for _, file := range fileInfo {
 		path := filepath.Join(h.scriptsDir, file.Name())
-		files.Add(path, path)
+		log.Printf("-- Including script: %s\n", path)
+		files.Add(path, filepath.Join(h.destPath, file.Name()))
 	}
 	return files, nil
 }

internal/filelist/initramfs/initramfs.go

@@ -9,6 +9,7 @@ import (
 // combining the output from them.
 type Initramfs struct {
 	features []filelist.FileLister
+	files    *filelist.FileList
 }
 
 // New returns a new Initramfs that generate a list of files based on the given
@@ -20,15 +21,18 @@ func New(features []filelist.FileLister) *Initramfs {
 }
 
 func (i *Initramfs) List() (*filelist.FileList, error) {
-	files := filelist.NewFileList()
+	if i.files != nil {
+		return i.files, nil
+	}
+	i.files = filelist.NewFileList()
 
 	for _, f := range i.features {
 		list, err := f.List()
 		if err != nil {
 			return nil, err
 		}
-		files.Import(list)
+		i.files.Import(list)
 	}
 
-	return files, nil
+	return i.files, nil
 }

internal/filelist/modules/modules.go

@@ -16,8 +16,8 @@ import (
 )
 
 type Modules struct {
-	modulesList     []string
 	modulesListPath string
+	modulesList     []string
 }
 
 // New returns a new Modules that will use the given moduleto provide a list
@@ -30,8 +30,6 @@ func New(modulesList []string, modulesListPath string) *Modules {
 }
 
 func (m *Modules) List() (*filelist.FileList, error) {
-	log.Println("- Including kernel modules")
-
 	kernVer, err := osutil.GetKernelVersion()
 	if err != nil {
 		return nil, err
@@ -55,20 +53,23 @@ func (m *Modules) List() (*filelist.FileList, error) {
 	}
 
 	// slurp up given list of modules
-	for _, module := range m.modulesList {
+	if len(m.modulesList) > 0 {
-		if modFilelist, err := getModule(module, modDir); err != nil {
+		log.Printf("-- Including kernel modules from deviceinfo")
-			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
+		for _, module := range m.modulesList {
-		} else {
+			if modFilelist, err := getModule(module, modDir); err != nil {
-			for _, file := range modFilelist {
+				return nil, fmt.Errorf("unable to get modules from deviceinfo: %w", err)
-				files.Add(file, file)
+			} else {
+				for _, file := range modFilelist {
+					files.Add(file, file)
+				}
 			}
 		}
 	}
 
 	// slurp up modules from lists in modulesListPath
+	log.Printf("- Searching for kernel modules from %s", m.modulesListPath)
 	fileInfo, err := os.ReadDir(m.modulesListPath)
 	if err != nil {
-		log.Println("-- Unable to find dir, skipping...")
 		return files, nil
 	}
 	for _, file := range fileInfo {
@@ -94,6 +95,9 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 	s := bufio.NewScanner(fd)
 	for s.Scan() {
 		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
 		dir, file := filepath.Split(line)
 		if file == "" {
 			// item is a directory
@@ -126,7 +130,11 @@ func slurpModules(fd io.Reader, modDir string) (*filelist.FileList, error) {
 }
 
 func getModulesInDir(modPath string) (files []string, err error) {
-	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
+	err = filepath.Walk(modPath, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			// Unable to walk path
+			return err
+		}
 		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
 			return nil
 		}
@@ -189,7 +197,12 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {
-		fields := strings.Fields(s.Text())
+		line := s.Text()
+		if len(line) == 0 || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		fields := strings.Fields(line)
 		if len(fields) == 0 {
 			continue
 		}

internal/filelist/osksdl/osksdl.go

@@ -37,6 +37,12 @@ func (s *OskSdl) List() (*filelist.FileList, error) {
 
 	log.Println("- Including osk-sdl support")
 
+	log.Println("******************* DEPRECATION WARNING *******************")
+	log.Println("Using osk-sdl is deprecated in postmarketOS!")
+	log.Println("Consider switching to unl0kr:")
+	log.Println("https://postmarketos.org/edge/2023/10/04/osk-sdl-deprecated/")
+	log.Println("******************* DEPRECATION WARNING *******************")
+
 	confFiles := []string{
 		"/etc/osk.conf",
 		"/etc/ts.conf",

internal/misc/misc.go

@@ -45,7 +45,7 @@ func RemoveDuplicates(in []string) (out []string) {
 //	defer misc.TimeFunc(time.Now(), "foo")
 func TimeFunc(start time.Time, name string) {
 	elapsed := time.Since(start)
-	log.Printf("%s completed in: %s", name, elapsed)
+	log.Printf("%s completed in: %.2fs", name, elapsed.Seconds())
 }
 
 // Exists tests if the given file/dir exists or not. Returns any errors related

pkgs/deviceinfo/deviceinfo.go

@@ -11,33 +11,43 @@ import (
 	"os"
 	"reflect"
 	"strings"
+
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 )
 
 type DeviceInfo struct {
-	InitfsCompression string
+	InitfsCompression      string
-	MesaDriver        string
+	InitfsExtraCompression string
-	ModulesInitfs     string
+	MesaDriver             string
-	UbootBoardname    string
+	ModulesInitfs          string
+	UbootBoardname         string
 }
 
-func ReadDeviceinfo(file string) (DeviceInfo, error) {
+// Reads the relevant entries from "file" into DeviceInfo struct
-	var deviceinfo DeviceInfo
+// Any already-set entries will be overwriten if they are present
+// in "file"
+func (d *DeviceInfo) ReadDeviceinfo(file string) error {
+	if exists, err := misc.Exists(file); !exists {
+		return fmt.Errorf("%q not found, required by mkinitfs", file)
+	} else if err != nil {
+		return fmt.Errorf("unexpected error getting status for %q: %s", file, err)
+	}
 
 	fd, err := os.Open(file)
 	if err != nil {
-		return deviceinfo, err
+		return err
 	}
 	defer fd.Close()
 
-	if err := unmarshal(fd, &deviceinfo); err != nil {
+	if err := d.unmarshal(fd); err != nil {
-		return deviceinfo, err
+		return err
 	}
 
-	return deviceinfo, nil
+	return nil
 }
 
 // Unmarshals a deviceinfo into a DeviceInfo struct
-func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
+func (d *DeviceInfo) unmarshal(r io.Reader) error {
 	s := bufio.NewScanner(r)
 	for s.Scan() {
 		line := s.Text()
@@ -73,7 +83,7 @@ func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
 			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
 		}
 
-		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
+		field := reflect.ValueOf(d).Elem().FieldByName(fieldName)
 		if !field.IsValid() {
 			// an option that meets the deviceinfo "specification", but isn't
 			// one we care about in this module

pkgs/deviceinfo/deviceinfo_test.go

@@ -10,6 +10,32 @@ import (
 	"testing"
 )
 
+// Test ReadDeviceinfo and the logic of reading from multiple files
+func TestReadDeviceinfo(t *testing.T) {
+	modules_expected := "panfrost foo bar bazz"
+	mesa_expected := "msm"
+
+	var devinfo DeviceInfo
+	err := devinfo.ReadDeviceinfo("./test_resources/deviceinfo-missing")
+	if !strings.Contains(err.Error(), "required by mkinitfs") {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-first")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	err = devinfo.ReadDeviceinfo("./test_resources/deviceinfo-msm")
+	if err != nil {
+		t.Errorf("received an unexpected err: %s", err)
+	}
+	if devinfo.ModulesInitfs != modules_expected {
+		t.Errorf("expected %q, got: %q", modules_expected, devinfo.ModulesInitfs)
+	}
+	if devinfo.MesaDriver != mesa_expected {
+		t.Errorf("expected %q, got: %q", mesa_expected, devinfo.MesaDriver)
+	}
+}
+
 // Test conversion of name to DeviceInfo struct field format
 func TestNameToField(t *testing.T) {
 	tables := []struct {
@@ -21,6 +47,7 @@ func TestNameToField(t *testing.T) {
 		{"deviceinfo_modules_initfs", "ModulesInitfs"},
 		{"modules_initfs", "ModulesInitfs"},
 		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+		{"deviceinfo_initfs_extra_compression", "InitfsExtraCompression"},
 	}
 
 	for _, table := range tables {
@@ -57,7 +84,7 @@ func TestUnmarshal(t *testing.T) {
 	var d DeviceInfo
 	for _, table := range tables {
 		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
-		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
+		if err := d.unmarshal(strings.NewReader(table.in)); err != nil {
 			t.Errorf("%s received an unexpected err: ", err)
 		}
 

pkgs/deviceinfo/test_resources/deviceinfo-first

@@ -0,0 +1,2 @@
+deviceinfo_modules_initfs="panfrost foo bar bazz"
+deviceinfo_mesa_driver="panfrost"

pkgs/deviceinfo/test_resources/deviceinfo-msm

@@ -0,0 +1 @@
+deviceinfo_mesa_driver="msm"