getInitfsFiles: fix path to kpartx

This changed in a multipath update.

Fixes #16

.gitlab-ci.yml

@@ -1,7 +1,10 @@
 ---
 
 # global settings
-image: alpine:latest
+image: alpine:edge
+
+variables:
+  GOFLAGS: "-buildvcs=false"
 
 stages:
   - lint
@@ -21,23 +24,13 @@ stages:
     - merge_requests
     - tags
 
-# device documentation
-gofmt linting:
-  stage: lint
-  allow_failure: true
-  <<: *only-default
-  before_script:
-    - apk -q add go
-  script:
-    - .gitlab-ci/check_gofmt.sh
-
 build:
   stage: build
   <<: *only-default
   before_script:
-    - apk -q add go
+    - apk -q add go staticcheck make
   script:
-    - go build -v
+    - make test
-    - go test ./...
+    - make
   artifacts:
     expire_in: 1 week

.gitlab-ci/check_gofmt.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-files="$(gofmt -l .)"
-
-[ -z "$files" ] && exit 0
-
-# run gofmt to print out the diff of what needs to be changed
-
-gofmt -d -e .
-
-exit 1

Makefile

@@ -0,0 +1,54 @@
+.POSIX:
+.SUFFIXES:
+
+PREFIX?=/usr/local
+BINDIR?=$(PREFIX)/sbin
+SHAREDIR?=$(PREFIX)/share
+GO?=go
+GOFLAGS?=
+LDFLAGS+=-s -w
+RM?=rm -f
+GOTEST=go test -count=1 -race
+
+GOSRC!=find * -name '*.go'
+GOSRC+=go.mod go.sum
+
+all: postmarketos-mkinitfs
+
+postmarketos-mkinitfs: $(GOSRC)
+	$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o postmarketos-mkinitfs
+
+.PHONY: fmt
+fmt:
+	gofmt -w .
+
+test:
+	@if [ `gofmt -l . | wc -l` -ne 0 ]; then \
+		gofmt -d .; \
+		echo "ERROR: source files need reformatting with gofmt"; \
+		exit 1; \
+	fi
+	@staticcheck ./...
+
+	@$(GOTEST) ./...
+
+clean:
+	$(RM) postmarketos-mkinitfs 
+
+install: $(DOCS) postmarketos-mkinitfs
+	install -Dm755 postmarketos-mkinitfs -t $(DESTDIR)$(BINDIR)/
+	ln -sf postmarketos-mkinitfs $(DESTDIR)$(BINDIR)/mkinitfs
+
+.PHONY: checkinstall
+checkinstall:
+	test -e $(DESTDIR)$(BINDIR)/postmarketos-mkinitfs
+	test -L $(DESTDIR)$(BINDIR)/mkinitfs
+
+RMDIR_IF_EMPTY:=sh -c '! [ -d $$0 ] || ls -1qA $$0 | grep -q . || rmdir $$0'
+
+uninstall:
+	$(RM) $(DESTDIR)$(BINDIR)/postmarketos-mkinitfs
+	$(RM) $(DESTDIR)$(BINDIR)/mkinitfs
+	${RMDIR_IF_EMPTY} $(DESTDIR)$(BINDIR)
+
+.PHONY: all clean install uninstall test

README.md

@@ -0,0 +1,45 @@
+`postmarketos-mkinitfs` is a tool for generating an initramfs (and installing
+it) on postmarketOS.
+
+## Building
+
+Building this project requires a Go compiler/toolchain and `make`:
+
+```
+$ make
+```
+
+To install locally:
+
+```
+$ make install
+```
+
+Installation prefix can be set in the generally accepted way with setting
+`PREFIX`:
+
+```
+$ make PREFIX=/some/location
+# make PREFIX=/some/location install
+```
+
+Other paths can be modified from the command line as well, see the top section of
+the `Makefile` for more information.
+
+Tests (functional and linting) can be executed by using the `test` make target:
+
+```
+$ make test
+```
+
+## Usage
+
+The application uses configuration from `/etc/deviceinfo`, and does not support
+any other options at runtime. It can be run simply by executing:
+
+```
+$ postmarketos-mkinitfs
+```
+
+For historical reasons, a symlink from `mkinitfs` to `postmarketos-mkinitfs` is
+also installed by the makefile's `install` target.

go.mod

@@ -3,8 +3,6 @@ module gitlab.com/postmarketOS/postmarketos-mkinitfs
 go 1.16
 
 require (
-	git.sr.ht/~sircmpwn/getopt v0.0.0-20201218204720-9961a9c6298f
-	github.com/BurntSushi/toml v0.4.0
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/klauspost/compress v1.13.3 // indirect
 	github.com/klauspost/pgzip v1.2.5

go.sum

@@ -1,30 +1,9 @@
-git.sr.ht/~sircmpwn/getopt v0.0.0-20201218204720-9961a9c6298f h1:f5axCdaRzGDCihN3o1Lq0ydn0VlkhY+11G0JOyY5qss=
-git.sr.ht/~sircmpwn/getopt v0.0.0-20201218204720-9961a9c6298f/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw=
-github.com/BurntSushi/toml v0.3.2-0.20210614224209-34d990aa228d/go.mod h1:2QZjSXA5e+XyFeCAxxtL8Z4StYUsTquL8ODGPR3C3MA=
-github.com/BurntSushi/toml v0.3.2-0.20210621044154-20a94d639b8e/go.mod h1:t4zg8TkHfP16Vb3x4WKIw7zVYMit5QFtPEO8lOWxzTg=
-github.com/BurntSushi/toml v0.3.2-0.20210624061728-01bfc69d1057/go.mod h1:NMj2lD5LfMqcE0w8tnqOsH6944oaqpI1974lrIwerfE=
-github.com/BurntSushi/toml v0.3.2-0.20210704081116-ccff24ee4463/go.mod h1:EkRrMiQQmfxK6kIldz3QbPlhmVkrjW1RDJUnbDqGYvc=
-github.com/BurntSushi/toml v0.4.0 h1:qD/r9AL67srjW6O3fcSKZDsXqzBNX6ieSRywr2hRrdE=
-github.com/BurntSushi/toml v0.4.0/go.mod h1:wtejDu7Q0FhCWAo2aXkywSJyYFg01EDTKozLNCz2JBA=
-github.com/BurntSushi/toml-test v0.1.1-0.20210620192437-de01089bbf76/go.mod h1:P/PrhmZ37t5llHfDuiouWXtFgqOoQ12SAh9j6EjrBR4=
-github.com/BurntSushi/toml-test v0.1.1-0.20210624055653-1f6389604dc6/go.mod h1:UAIt+Eo8itMZAAgImXkPGDMYsT1SsJkVdB5TuONl86A=
-github.com/BurntSushi/toml-test v0.1.1-0.20210704062846-269931e74e3f/go.mod h1:fnFWrIwqgHsEjVsW3RYCJmDo86oq9eiJ9u6bnqhtm2g=
-github.com/BurntSushi/toml-test v0.1.1-0.20210723065233-facb9eccd4da h1:2QGUaQtV2u8V1USTI883wo+uxtZFAiZ4TCNupHJ98IU=
-github.com/BurntSushi/toml-test v0.1.1-0.20210723065233-facb9eccd4da/go.mod h1:ve9Q/RRu2vHi42LocPLNvagxuUJh993/95b18bw/Nws=
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RSSp2Om9lubZpiMgVbvn39bsUmW9U5h0twqc=
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ=
 github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-zgo.at/zli v0.0.0-20210619044753-e7020a328e59/go.mod h1:HLAc12TjNGT+VRXr76JnsNE3pbooQtwKWhX+RlDjQ2Y=

main.go

@@ -1,22 +1,23 @@
-// Copyright 2021 Clayton Craft <clayton@craftyguy.net>
+// Copyright 2022 Clayton Craft <clayton@craftyguy.net>
 // SPDX-License-Identifier: GPL-3.0-or-later
+
 package main
 
 import (
 	"bufio"
 	"debug/elf"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
-	"git.sr.ht/~sircmpwn/getopt"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/archive"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/misc"
@@ -28,21 +29,22 @@ func timeFunc(start time.Time, name string) {
 }
 
 func main() {
-	devinfo, err := deviceinfo.ReadDeviceinfo()
+	deviceinfoFile := "/etc/deviceinfo"
-	if err != nil {
+	if !exists(deviceinfoFile) {
 		log.Print("NOTE: deviceinfo (from device package) not installed yet, " +
 			"not building the initramfs now (it should get built later " +
 			"automatically.)")
-		os.Exit(0)
+		return
 	}
 
-	var outDir string
+	devinfo, err := deviceinfo.ReadDeviceinfo(deviceinfoFile)
-	getopt.StringVar(&outDir, "d", "/boot", "Directory to output initfs(-extra) and other boot files, default: /boot")
+	if err != nil {
-
-	if err := getopt.Parse(); err != nil {
 		log.Fatal(err)
 	}
 
+	outDir := flag.String("d", "/boot", "Directory to output initfs(-extra) and other boot files")
+	flag.Parse()
+
 	defer timeFunc(time.Now(), "mkinitfs")
 
 	kernVer, err := getKernelVersion()
@@ -50,31 +52,35 @@ func main() {
 		log.Fatal(err)
 	}
 
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	// temporary working dir
-	workDir, err := ioutil.TempDir("", "mkinitfs")
+	workDir, err := os.MkdirTemp("", "mkinitfs")
 	if err != nil {
 		log.Fatal("Unable to create temporary work directory:", err)
 	}
 	defer os.RemoveAll(workDir)
 
 	log.Print("Generating for kernel version: ", kernVer)
-	log.Print("Output directory: ", outDir)
+	log.Print("Output directory: ", *outDir)
 
 	if err := generateInitfs("initramfs", workDir, kernVer, devinfo); err != nil {
-		log.Fatal(err)
+		log.Fatal("generateInitfs: ", err)
 	}
 
 	if err := generateInitfsExtra("initramfs-extra", workDir, devinfo); err != nil {
-		log.Fatal(err)
+		log.Fatal("generateInitfsExtra: ", err)
+	}
+
+	if err := copyUbootFiles(workDir, devinfo); errors.Is(err, os.ErrNotExist) {
+		log.Println("u-boot files copying skipped: ", err)
+	} else {
+		if err != nil {
+			log.Fatal("copyUbootFiles: ", err)
+		}
 	}
 
 	// Final processing of initramfs / kernel is done by boot-deploy
-	if err := bootDeploy(workDir, outDir); err != nil {
+	if err := bootDeploy(workDir, *outDir); err != nil {
-		log.Fatal(err)
+		log.Fatal("bootDeploy: ", err)
 	}
 
 }
@@ -87,18 +93,29 @@ func bootDeploy(workDir string, outDir string) error {
 	if len(kernels) == 0 {
 		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
 	}
-	kernFile, err := os.Open(kernels[0])
+
+	// Pick a kernel that does not have suffixes added by boot-deploy
+	var kernFile string
+	for _, f := range kernels {
+		if strings.HasSuffix(f, "-dtb") || strings.HasSuffix(f, "-mtk") {
+			continue
+		}
+		kernFile = f
+		break
+	}
+
+	kernFd, err := os.Open(kernFile)
 	if err != nil {
 		return err
 	}
-	defer kernFile.Close()
+	defer kernFd.Close()
 
 	kernFileCopy, err := os.Create(filepath.Join(workDir, "vmlinuz"))
 	if err != nil {
 		return err
 	}
 
-	if _, err = io.Copy(kernFileCopy, kernFile); err != nil {
+	if _, err = io.Copy(kernFileCopy, kernFd); err != nil {
 		return err
 	}
 	kernFileCopy.Close()
@@ -111,33 +128,19 @@ func bootDeploy(workDir string, outDir string) error {
 		"-o", outDir,
 		"initramfs-extra")
 	if !exists(cmd.Path) {
-		return errors.New("boot-deploy command not found.")
+		return errors.New("boot-deploy command not found")
 	}
 
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	// err is ignored, since shellcheck will return != 0 if there are issues
 	if err := cmd.Run(); err != nil {
-		log.Print("'boot-deploy' command failed: ")
+		log.Print("'boot-deploy' command failed")
 		return err
 	}
 
 	return nil
 }
 
-func createInitfsRootDirs(initfsRoot string) {
-	dirs := []string{
-		"/bin", "/sbin", "/usr/bin", "/usr/lib", "/usr/sbin", "/proc", "/sys",
-		"/dev", "/tmp", "/lib", "/boot", "/sysroot", "/etc",
-	}
-
-	for _, dir := range dirs {
-		if err := os.MkdirAll(filepath.Join(initfsRoot, dir), os.FileMode(0775)); err != nil {
-			log.Fatal(err)
-		}
-	}
-}
-
 func exists(file string) bool {
 	if _, err := os.Stat(file); err == nil {
 		return true
@@ -145,72 +148,74 @@ func exists(file string) bool {
 	return false
 }
 
-func getHookFiles(filesdir string) misc.StringSet {
+func getHookFiles(filesdir string) (files []string, err error) {
-	fileInfo, err := ioutil.ReadDir(filesdir)
+	fileInfo, err := os.ReadDir(filesdir)
 	if err != nil {
-		log.Fatal(err)
+		return nil, fmt.Errorf("getHookFiles: unable to read hook file dir: %w", err)
 	}
-	files := make(misc.StringSet)
 	for _, file := range fileInfo {
 		path := filepath.Join(filesdir, file.Name())
 		f, err := os.Open(path)
 		if err != nil {
-			log.Fatal(err)
+			return nil, fmt.Errorf("getHookFiles: unable to open hook file: %w", err)
+
 		}
 		defer f.Close()
+		log.Printf("-- Including files from: %s\n", path)
 		s := bufio.NewScanner(f)
 		for s.Scan() {
-			if !exists(s.Text()) {
+			if filelist, err := getFiles([]string{s.Text()}, true); err != nil {
-				log.Fatalf("Unable to find file %q required by %q", s.Text(), path)
+				return nil, fmt.Errorf("getHookFiles: unable to find file %q required by %q", s.Text(), path)
+			} else {
+				files = append(files, filelist...)
 			}
-			files[s.Text()] = false
 		}
 		if err := s.Err(); err != nil {
-			log.Fatal(err)
+			return nil, fmt.Errorf("getHookFiles: uname to process hook file %q: %w", path, err)
 		}
 	}
-	return files
+	return files, nil
 }
 
 // Recursively list all dependencies for a given ELF binary
-func getBinaryDeps(files misc.StringSet, file string) error {
+func getBinaryDeps(file string) (files []string, err error) {
 	// if file is a symlink, resolve dependencies for target
 	fileStat, err := os.Lstat(file)
 	if err != nil {
-		log.Print("getBinaryDeps: failed to stat file")
+		return nil, fmt.Errorf("getBinaryDeps: failed to stat file %q: %w", file, err)
-		return err
 	}
 
 	// Symlink: write symlink to archive then set 'file' to link target
 	if fileStat.Mode()&os.ModeSymlink != 0 {
 		target, err := os.Readlink(file)
 		if err != nil {
-			log.Print("getBinaryDeps: unable to read symlink: ", file)
+			return nil, fmt.Errorf("getBinaryDeps: unable to read symlink %q: %w", file, err)
-			return err
 		}
 		if !filepath.IsAbs(target) {
 			target, err = misc.RelativeSymlinkTargetToDir(target, filepath.Dir(file))
 			if err != nil {
-				return err
+				return files, err
 			}
 		}
-		if err := getBinaryDeps(files, target); err != nil {
+		binaryDepFiles, err := getBinaryDeps(target)
-			return err
+		if err != nil {
+			return files, err
 		}
-		return err
+		files = append(files, binaryDepFiles...)
+		return files, err
 	}
 
 	// get dependencies for binaries
 	fd, err := elf.Open(file)
 	if err != nil {
-		log.Fatal(err)
+		return nil, fmt.Errorf("getBinaryDeps: unable to open elf binary %q: %w", file, err)
 	}
 	libs, _ := fd.ImportedLibraries()
 	fd.Close()
-	files[file] = false
+	files = append(files, file)
 
 	if len(libs) == 0 {
-		return err
+		return files, err
 	}
 
 	libdirs := []string{"/usr/lib", "/lib"}
@@ -219,55 +224,96 @@ func getBinaryDeps(files misc.StringSet, file string) error {
 		for _, libdir := range libdirs {
 			path := filepath.Join(libdir, lib)
 			if _, err := os.Stat(path); err == nil {
-				err := getBinaryDeps(files, path)
+				binaryDepFiles, err := getBinaryDeps(path)
 				if err != nil {
-					return err
+					return files, err
 				}
-				files[path] = false
+				files = append(files, binaryDepFiles...)
+				files = append(files, path)
 				found = true
 				break
 			}
 		}
 		if !found {
-			log.Fatalf("Unable to locate dependency for %q: %s", file, lib)
+			return nil, fmt.Errorf("getBinaryDeps: unable to locate dependency for %q: %s", file, lib)
 		}
 	}
 
-	return nil
+	return
 }
 
-func getFiles(files misc.StringSet, newFiles misc.StringSet, required bool) error {
+func getFiles(list []string, required bool) (files []string, err error) {
-	for file := range newFiles {
+	for _, file := range list {
-		err := getFile(files, file, required)
+		filelist, err := getFile(file, required)
 		if err != nil {
-			return err
+			return nil, err
 		}
+		files = append(files, filelist...)
 	}
-	return nil
+
+	files = misc.RemoveDuplicates(files)
+	return
 }
 
-func getFile(files misc.StringSet, file string, required bool) error {
+func getFile(file string, required bool) (files []string, err error) {
-	if !exists(file) {
+	// Expand glob expression
-		if required {
+	expanded, err := filepath.Glob(file)
-			return errors.New("getFile: File does not exist :" + file)
-		}
-		return nil
-	}
-
-	files[file] = false
-
-	// get dependencies for binaries
-	if _, err := elf.Open(file); err != nil {
-		// file is not an elf, so don't resolve lib dependencies
-		return nil
-	}
-
-	err := getBinaryDeps(files, file)
 	if err != nil {
-		return err
+		return
+	}
+	if len(expanded) > 0 && expanded[0] != file {
+		for _, path := range expanded {
+			if globFiles, err := getFile(path, required); err != nil {
+				return files, err
+			} else {
+				files = append(files, globFiles...)
+			}
+		}
+		return misc.RemoveDuplicates(files), nil
 	}
 
-	return nil
+	fileInfo, err := os.Stat(file)
+	if err != nil {
+		if required {
+			return files, errors.New("getFile: File does not exist :" + file)
+		}
+		return files, nil
+	}
+
+	if fileInfo.IsDir() {
+		// Recurse over directory contents
+		err := filepath.Walk(file, func(path string, f os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if f.IsDir() {
+				return nil
+			}
+			newFiles, err := getFile(path, required)
+			if err != nil {
+				return err
+			}
+			files = append(files, newFiles...)
+			return nil
+		})
+		if err != nil {
+			return files, err
+		}
+	} else {
+		files = append(files, file)
+
+		// get dependencies for binaries
+		if _, err := elf.Open(file); err == nil {
+			if binaryDepFiles, err := getBinaryDeps(file); err != nil {
+				return files, err
+			} else {
+				files = append(files, binaryDepFiles...)
+			}
+		}
+	}
+
+	files = misc.RemoveDuplicates(files)
+	return
 }
 
 func getOskConfFontPath(oskConfPath string) (string, error) {
@@ -294,164 +340,199 @@ func getOskConfFontPath(oskConfPath string) (string, error) {
 
 // Get a list of files and their dependencies related to supporting rootfs full
 // disk (d)encryption
-func getFdeFiles(files misc.StringSet, devinfo deviceinfo.DeviceInfo) error {
+func getFdeFiles(devinfo deviceinfo.DeviceInfo) (files []string, err error) {
-	confFiles := misc.StringSet{
+	confFiles := []string{
-		"/etc/osk.conf":   false,
+		"/etc/osk.conf",
-		"/etc/ts.conf":    false,
+		"/etc/ts.conf",
-		"/etc/pointercal": false,
+		"/etc/pointercal",
-		"/etc/fb.modes":   false,
+		"/etc/fb.modes",
-		"/etc/directfbrc": false,
+		"/etc/directfbrc",
 	}
 	// TODO: this shouldn't be false? though some files (pointercal) don't always exist...
-	if err := getFiles(files, confFiles, false); err != nil {
+	if files, err = getFiles(confFiles, false); err != nil {
-		return err
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
 	}
 
 	// osk-sdl
-	oskFiles := misc.StringSet{
+	oskFiles := []string{
-		"/usr/bin/osk-sdl":    false,
+		"/usr/bin/osk-sdl",
-		"/sbin/cryptsetup":    false,
+		"/sbin/cryptsetup",
-		"/usr/lib/libGL.so.1": false}
+		"/usr/lib/libGL.so.1",
-	if err := getFiles(files, oskFiles, true); err != nil {
+	}
-		return err
+	if filelist, err := getFiles(oskFiles, true); err != nil {
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	} else {
+		files = append(files, filelist...)
 	}
 
 	fontFile, err := getOskConfFontPath("/etc/osk.conf")
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("getFdeFiles: failed to add file %q: %w", fontFile, err)
 	}
-	files[fontFile] = false
+	files = append(files, fontFile)
 
 	// Directfb
-	dfbFiles := make(misc.StringSet)
+	dfbFiles := []string{}
 	err = filepath.Walk("/usr/lib/directfb-1.7-7", func(path string, f os.FileInfo, err error) error {
 		if filepath.Ext(path) == ".so" {
-			dfbFiles[path] = false
+			dfbFiles = append(dfbFiles, path)
 		}
 		return nil
 	})
 	if err != nil {
-		log.Print("getBinaryDeps: failed to stat file")
+		return nil, fmt.Errorf("getFdeFiles: failed to add file %w", err)
-		return err
 	}
-	if err := getFiles(files, dfbFiles, true); err != nil {
+	if filelist, err := getFiles(dfbFiles, true); err != nil {
-		return err
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+	} else {
+		files = append(files, filelist...)
 	}
 
 	// tslib
-	tslibFiles := make(misc.StringSet)
+	tslibFiles := []string{}
 	err = filepath.Walk("/usr/lib/ts", func(path string, f os.FileInfo, err error) error {
 		if filepath.Ext(path) == ".so" {
-			tslibFiles[path] = false
+			tslibFiles = append(tslibFiles, path)
 		}
 		return nil
 	})
 	if err != nil {
-		log.Print("getBinaryDeps: failed to stat file")
+		return nil, fmt.Errorf("getFdeFiles: failed to add file: %w", err)
-		return err
 	}
 	libts, _ := filepath.Glob("/usr/lib/libts*")
-	for _, file := range libts {
+	tslibFiles = append(tslibFiles, libts...)
-		tslibFiles[file] = false
+	if filelist, err := getFiles(tslibFiles, true); err != nil {
-	}
+		return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
-	if err = getFiles(files, tslibFiles, true); err != nil {
+	} else {
-		return err
+		files = append(files, filelist...)
 	}
 
 	// mesa hw accel
-	if devinfo.Deviceinfo_mesa_driver != "" {
+	if devinfo.MesaDriver != "" {
-		mesaFiles := misc.StringSet{
+		mesaFiles := []string{
-			"/usr/lib/libEGL.so.1":    false,
+			"/usr/lib/libEGL.so.1",
-			"/usr/lib/libGLESv2.so.2": false,
+			"/usr/lib/libGLESv2.so.2",
-			"/usr/lib/libgbm.so.1":    false,
+			"/usr/lib/libgbm.so.1",
-			"/usr/lib/libudev.so.1":   false,
+			"/usr/lib/libudev.so.1",
-			"/usr/lib/xorg/modules/dri/" + devinfo.Deviceinfo_mesa_driver + "_dri.so": false,
+			"/usr/lib/xorg/modules/dri/" + devinfo.MesaDriver + "_dri.so",
 		}
-		if err := getFiles(files, mesaFiles, true); err != nil {
+		if filelist, err := getFiles(mesaFiles, true); err != nil {
-			return err
+			return nil, fmt.Errorf("getFdeFiles: failed to add files: %w", err)
+		} else {
+			files = append(files, filelist...)
 		}
 	}
 
-	return nil
+	return
 }
 
-func getHookScripts(files misc.StringSet) {
+func getHookScripts() (files []string) {
 	scripts, _ := filepath.Glob("/etc/postmarketos-mkinitfs/hooks/*.sh")
-	for _, script := range scripts {
+	files = append(files, scripts...)
-		files[script] = false
+
-	}
+	return
 }
 
-func getInitfsExtraFiles(files misc.StringSet, devinfo deviceinfo.DeviceInfo) error {
+func getInitfsExtraFiles(devinfo deviceinfo.DeviceInfo) (files []string, err error) {
 	log.Println("== Generating initramfs extra ==")
-	binariesExtra := misc.StringSet{
+	binariesExtra := []string{
-		"/lib/libz.so.1":        false,
+		"/lib/libz.so.1",
-		"/sbin/dmsetup":         false,
+		"/sbin/btrfs",
-		"/sbin/e2fsck":          false,
+		"/sbin/dmsetup",
-		"/usr/sbin/parted":      false,
+		"/sbin/e2fsck",
-		"/usr/sbin/resize2fs":   false,
+		"/usr/sbin/parted",
-		"/usr/sbin/resize.f2fs": false,
+		"/usr/sbin/resize2fs",
+		"/usr/sbin/resize.f2fs",
 	}
 	log.Println("- Including extra binaries")
-	if err := getFiles(files, binariesExtra, true); err != nil {
+	if filelist, err := getFiles(binariesExtra, true); err != nil {
-		return err
+		return nil, err
+	} else {
+		files = append(files, filelist...)
+	}
+
+	// Hook files & scripts
+	if exists("/etc/postmarketos-mkinitfs/files-extra") {
+		log.Println("- Including hook files")
+		var hookFiles []string
+		hookFiles, err := getHookFiles("/etc/postmarketos-mkinitfs/files-extra")
+		if err != nil {
+			return nil, err
+		}
+		if filelist, err := getFiles(hookFiles, true); err != nil {
+			return nil, err
+		} else {
+			files = append(files, filelist...)
+		}
 	}
 
 	if exists("/usr/bin/osk-sdl") {
 		log.Println("- Including FDE support")
-		if err := getFdeFiles(files, devinfo); err != nil {
+		if fdeFiles, err := getFdeFiles(devinfo); err != nil {
-			return err
+			return nil, err
+		} else {
+			files = append(files, fdeFiles...)
 		}
 	} else {
 		log.Println("- *NOT* including FDE support")
 	}
 
-	return nil
+	return
 }
 
-func getInitfsFiles(files misc.StringSet, devinfo deviceinfo.DeviceInfo) error {
+func getInitfsFiles(devinfo deviceinfo.DeviceInfo) (files []string, err error) {
 	log.Println("== Generating initramfs ==")
-	requiredFiles := misc.StringSet{
+	requiredFiles := []string{
-		"/bin/busybox":        false,
+		"/bin/busybox",
-		"/bin/sh":             false,
+		"/bin/sh",
-		"/bin/busybox-extras": false,
+		"/bin/busybox-extras",
-		"/usr/sbin/telnetd":   false,
+		"/usr/sbin/telnetd",
-		"/sbin/kpartx":        false,
+		"/usr/sbin/kpartx",
-		"/etc/deviceinfo":     false,
+		"/etc/deviceinfo",
+		"/usr/bin/unudhcpd",
 	}
 
 	// Hook files & scripts
 	if exists("/etc/postmarketos-mkinitfs/files") {
 		log.Println("- Including hook files")
-		hookFiles := getHookFiles("/etc/postmarketos-mkinitfs/files")
+		if hookFiles, err := getHookFiles("/etc/postmarketos-mkinitfs/files"); err != nil {
-		if err := getFiles(files, hookFiles, true); err != nil {
+			return nil, err
-			return err
+		} else {
+			if filelist, err := getFiles(hookFiles, true); err != nil {
+				return nil, err
+			} else {
+				files = append(files, filelist...)
+			}
 		}
 	}
+
 	log.Println("- Including hook scripts")
-	getHookScripts(files)
+	hookScripts := getHookScripts()
+	files = append(files, hookScripts...)
 
 	log.Println("- Including required binaries")
-	if err := getFiles(files, requiredFiles, true); err != nil {
+	if filelist, err := getFiles(requiredFiles, true); err != nil {
-		return err
+		return nil, err
+	} else {
+		files = append(files, filelist...)
 	}
 
-	return nil
+	return
 }
 
-func getInitfsModules(files misc.StringSet, devinfo deviceinfo.DeviceInfo, kernelVer string) error {
+func getInitfsModules(devinfo deviceinfo.DeviceInfo, kernelVer string) (files []string, err error) {
 	log.Println("- Including kernel modules")
 
 	modDir := filepath.Join("/lib/modules", kernelVer)
 	if !exists(modDir) {
-		return errors.New("Kernel module directory not found: " + modDir)
+		// dir /lib/modules/<kernel> if kernel built without module support, so just print a message
+		log.Printf("-- kernel module directory not found: %q, not including modules", modDir)
+		return
 	}
 
 	// modules.* required by modprobe
 	modprobeFiles, _ := filepath.Glob(filepath.Join(modDir, "modules.*"))
-	for _, file := range modprobeFiles {
+	files = append(files, modprobeFiles...)
-		files[file] = false
-	}
 
 	// module name (without extension), or directory (trailing slash is important! globs OK)
 	requiredModules := []string{
@@ -469,29 +550,30 @@ func getInitfsModules(files misc.StringSet, devinfo deviceinfo.DeviceInfo, kerne
 			dir = filepath.Join(modDir, dir)
 			dirs, _ := filepath.Glob(dir)
 			for _, d := range dirs {
-				if err := getModulesInDir(files, d); err != nil {
+				if filelist, err := getModulesInDir(d); err != nil {
-					log.Print("Unable to get modules in dir: ", d)
+					return nil, fmt.Errorf("getInitfsModules: unable to get modules dir %q: %w", d, err)
-					return err
+				} else {
+					files = append(files, filelist...)
 				}
 			}
-			continue
 		} else if dir == "" {
 			// item is a module name
-			if err := getModule(files, file, modDir); err != nil {
+			if filelist, err := getModule(file, modDir); err != nil {
-				log.Print("Unable to get module: ", file)
+				return nil, fmt.Errorf("getInitfsModules: unable to get module %q: %w", file, err)
-				return err
+			} else {
+				files = append(files, filelist...)
 			}
-			continue
 		} else {
 			log.Printf("Unknown module entry: %q", item)
 		}
 	}
 
 	// deviceinfo modules
-	for _, module := range strings.Fields(devinfo.Deviceinfo_modules_initfs) {
+	for _, module := range strings.Fields(devinfo.ModulesInitfs) {
-		if err := getModule(files, module, modDir); err != nil {
+		if filelist, err := getModule(module, modDir); err != nil {
-			log.Print("Unable to get modules from deviceinfo")
+			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
-			return err
+		} else {
+			files = append(files, filelist...)
 		}
 	}
 
@@ -500,27 +582,27 @@ func getInitfsModules(files misc.StringSet, devinfo deviceinfo.DeviceInfo, kerne
 	for _, modFile := range initfsModFiles {
 		f, err := os.Open(modFile)
 		if err != nil {
-			log.Print("getInitfsModules: unable to open mkinitfs modules file: ", modFile)
+			return nil, fmt.Errorf("getInitfsModules: unable to open mkinitfs modules file %q: %w", modFile, err)
-			return err
 		}
 		defer f.Close()
 		s := bufio.NewScanner(f)
 		for s.Scan() {
-			if err := getModule(files, s.Text(), modDir); err != nil {
+			if filelist, err := getModule(s.Text(), modDir); err != nil {
-				log.Print("getInitfsModules: unable to get module file: ", s.Text())
+				return nil, fmt.Errorf("getInitfsModules: unable to get module file %q: %w", s.Text(), err)
-				return err
+			} else {
+				files = append(files, filelist...)
 			}
 		}
 	}
 
-	return nil
+	return
 }
 
 func getKernelReleaseFile() (string, error) {
 	files, _ := filepath.Glob("/usr/share/kernel/*/kernel.release")
 	// only one kernel flavor supported
 	if len(files) != 1 {
-		return "", errors.New(fmt.Sprintf("Only one kernel release/flavor is supported, found: %q", files))
+		return "", fmt.Errorf("only one kernel release/flavor is supported, found: %q", files)
 	}
 
 	return files[0], nil
@@ -542,6 +624,50 @@ func getKernelVersion() (string, error) {
 	return strings.TrimSpace(string(contents)), nil
 }
 
+func Copy(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func copyUbootFiles(path string, devinfo deviceinfo.DeviceInfo) error {
+	if devinfo.UbootBoardname == "" {
+		return nil
+	}
+
+	srcDir := filepath.Join("/usr/share/u-boot", devinfo.UbootBoardname)
+	entries, err := os.ReadDir(srcDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(srcDir, entry.Name())
+		destPath := filepath.Join(path, entry.Name())
+
+		if err := Copy(sourcePath, destPath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func generateInitfs(name string, path string, kernVer string, devinfo deviceinfo.DeviceInfo) error {
 	initfsArchive, err := archive.New()
 	if err != nil {
@@ -553,18 +679,40 @@ func generateInitfs(name string, path string, kernVer string, devinfo deviceinfo
 		"/dev", "/tmp", "/lib", "/boot", "/sysroot", "/etc",
 	}
 	for _, dir := range requiredDirs {
-		initfsArchive.Dirs[dir] = false
+		if err := initfsArchive.AddItem(dir, dir); err != nil {
+			return err
+		}
 	}
 
-	if err := getInitfsFiles(initfsArchive.Files, devinfo); err != nil {
+	if files, err := getInitfsFiles(devinfo); err != nil {
 		return err
+	} else {
+		items := make(map[string]string)
+		// copy files into a map, where the source(key) and dest(value) are the
+		// same
+		for _, f := range files {
+			items[f] = f
+		}
+		if err := initfsArchive.AddItems(items); err != nil {
+			return err
+		}
 	}
 
-	if err := getInitfsModules(initfsArchive.Files, devinfo, kernVer); err != nil {
+	if files, err := getInitfsModules(devinfo, kernVer); err != nil {
 		return err
+	} else {
+		items := make(map[string]string)
+		// copy files into a map, where the source(key) and dest(value) are the
+		// same
+		for _, f := range files {
+			items[f] = f
+		}
+		if err := initfsArchive.AddItems(items); err != nil {
+			return err
+		}
 	}
 
-	if err := initfsArchive.AddFile("/usr/share/postmarketos-mkinitfs/init.sh", "/init"); err != nil {
+	if err := initfsArchive.AddItem("/usr/share/postmarketos-mkinitfs/init.sh", "/init"); err != nil {
 		return err
 	}
 
@@ -573,13 +721,13 @@ func generateInitfs(name string, path string, kernVer string, devinfo deviceinfo
 	splashFiles, _ := filepath.Glob("/usr/share/postmarketos-splashes/*.ppm.gz")
 	for _, file := range splashFiles {
 		// splash images are expected at /<file>
-		if err := initfsArchive.AddFile(file, filepath.Join("/", filepath.Base(file))); err != nil {
+		if err := initfsArchive.AddItem(file, filepath.Join("/", filepath.Base(file))); err != nil {
 			return err
 		}
 	}
 
 	// initfs_functions
-	if err := initfsArchive.AddFile("/usr/share/postmarketos-mkinitfs/init_functions.sh", "/init_functions.sh"); err != nil {
+	if err := initfsArchive.AddItem("/usr/share/postmarketos-mkinitfs/init_functions.sh", "/init_functions.sh"); err != nil {
 		return err
 	}
 
@@ -597,8 +745,19 @@ func generateInitfsExtra(name string, path string, devinfo deviceinfo.DeviceInfo
 		return err
 	}
 
-	if err := getInitfsExtraFiles(initfsExtraArchive.Files, devinfo); err != nil {
+	if files, err := getInitfsExtraFiles(devinfo); err != nil {
 		return err
+	} else {
+
+		items := make(map[string]string)
+		// copy files into a map, where the source(key) and dest(value) are the
+		// same
+		for _, f := range files {
+			items[f] = f
+		}
+		if err := initfsExtraArchive.AddItems(items); err != nil {
+			return err
+		}
 	}
 
 	log.Println("- Writing and verifying initramfs-extra archive")
@@ -610,29 +769,23 @@ func generateInitfsExtra(name string, path string, devinfo deviceinfo.DeviceInfo
 }
 
 func stripExts(file string) string {
-	for {
+	return strings.Split(file, ".")[0]
-		if filepath.Ext(file) == "" {
-			break
-		}
-		file = strings.TrimSuffix(file, filepath.Ext(file))
-	}
-	return file
 }
 
-func getModulesInDir(files misc.StringSet, modPath string) error {
+func getModulesInDir(modPath string) (files []string, err error) {
-	err := filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
+	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
 		// TODO: need to support more extensions?
 		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
 			return nil
 		}
-		files[path] = false
+		files = append(files, path)
 		return nil
 	})
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return
 }
 
 // Given a module name, e.g. 'dwc_wdt', resolve the full path to the module
@@ -641,62 +794,56 @@ func getModulesInDir(files misc.StringSet, modPath string) error {
 // have been built into the kernel
 // TODO: look for it in modules.builtin, and make it fatal if it can't be found
 // anywhere
-func getModule(files misc.StringSet, modName string, modDir string) error {
+func getModule(modName string, modDir string) (files []string, err error) {
 
-	deps, err := getModuleDeps(modName, modDir)
+	modDep := filepath.Join(modDir, "modules.dep")
-	if err != nil {
+	if !exists(modDep) {
-		return err
+		return nil, fmt.Errorf("kernel module.dep not found: %s", modDir)
 	}
 
-	if len(deps) == 0 {
+	fd, err := os.Open(modDep)
-		// retry and swap - and _ in module name
+	if err != nil {
-		if strings.Contains(modName, "-") {
+		return nil, fmt.Errorf("unable to open modules.dep: %w", err)
-			modName = strings.ReplaceAll(modName, "-", "_")
+	}
-		} else {
+	defer fd.Close()
-			modName = strings.ReplaceAll(modName, "_", "-")
+
-		}
+	deps, err := getModuleDeps(modName, fd)
-		deps, err = getModuleDeps(modName, modDir)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return err
-		}
 	}
 
 	for _, dep := range deps {
 		p := filepath.Join(modDir, dep)
 		if !exists(p) {
-			log.Print(fmt.Sprintf("Tried to include a module that doesn't exist in the modules directory (%s): %s", modDir, p))
+			return nil, fmt.Errorf("tried to include a module that doesn't exist in the modules directory (%s): %s", modDir, p)
-			return err
 		}
-		files[p] = false
+		files = append(files, p)
 	}
 
-	return err
+	return
 }
 
-func getModuleDeps(modName string, modDir string) ([]string, error) {
+// Get the canonicalized name for the module as represented in the given modules.dep io.reader
+func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 	var deps []string
 
-	modDep := filepath.Join(modDir, "modules.dep")
+	// split the module name on - and/or _, build a regex for matching
-	if !exists(modDep) {
+	splitRe := regexp.MustCompile("[-_]+")
-		log.Fatal("Kernel module.dep not found: ", modDir)
+	modNameReStr := splitRe.ReplaceAllString(modName, "[-_]+")
-	}
+	re := regexp.MustCompile("^" + modNameReStr + "$")
 
-	fd, err := os.Open(modDep)
+	s := bufio.NewScanner(modulesDep)
-	if err != nil {
-		log.Print("Unable to open modules.dep: ", modDep)
-		return deps, err
-	}
-
-	defer fd.Close()
-	s := bufio.NewScanner(fd)
 	for s.Scan() {
 		fields := strings.Fields(s.Text())
-		fields[0] = strings.TrimSuffix(fields[0], ":")
+		if len(fields) == 0 {
-		if modName != filepath.Base(stripExts(fields[0])) {
 			continue
 		}
-		for _, modPath := range fields {
+		fields[0] = strings.TrimSuffix(fields[0], ":")
-			deps = append(deps, modPath)
+
+		found := re.FindAll([]byte(filepath.Base(stripExts(fields[0]))), -1)
+		if len(found) > 0 {
+			deps = append(deps, fields...)
+			break
 		}
 	}
 	if err := s.Err(); err != nil {

main_test.go

@@ -1,8 +1,10 @@
 // Copyright 2021 Clayton Craft <clayton@craftyguy.net>
 // SPDX-License-Identifier: GPL-3.0-or-later
+
 package main
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -24,3 +26,57 @@ func TestStripExts(t *testing.T) {
 		}
 	}
 }
+
+func stringSlicesEqual(a []string, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+var testModuleDep string = `
+kernel/sound/soc/codecs/snd-soc-msm8916-digital.ko:
+kernel/net/sched/act_ipt.ko.xz: kernel/net/netfilter/x_tables.ko.xz
+kernel/drivers/watchdog/watchdog.ko.xz:
+kernel/drivers/usb/serial/ir-usb.ko.xz: kernel/drivers/usb/serial/usbserial.ko.xz
+kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz:
+kernel/drivers/hid/hid-alps.ko.xz:
+kernel/net/netfilter/xt_u32.ko.xz: kernel/net/netfilter/x_tables.ko.xz
+kernel/net/netfilter/xt_sctp.ko.xz: kernel/net/netfilter/x_tables.ko.xz
+kernel/drivers/hwmon/gl518sm.ko.xz:
+kernel/drivers/watchdog/dw_wdt.ko.xz: kernel/drivers/watchdog/watchdog.ko.xz
+kernel/net/bluetooth/hidp/hidp.ko.xz: kernel/net/bluetooth/bluetooth.ko.xz kernel/net/rfkill/rfkill.ko.xz kernel/crypto/ecdh_generic.ko.xz kernel/crypto/ecc.ko.xz
+kernel/fs/nls/nls_iso8859-1.ko.xz:
+kernel/net/vmw_vsock/vmw_vsock_virtio_transport.ko.xz: kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko.xz kernel/drivers/virtio/virtio.ko.xz kernel/drivers/virtio/virtio_ring.ko.xz kernel/net/vmw_vsock/vsock.ko.xz
+kernel/drivers/gpu/drm/panfrost/panfrost.ko.xz: kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz
+kernel/drivers/gpu/drm/msm/msm.ko: kernel/drivers/gpu/drm/drm_kms_helper.ko
+`
+
+func TestGetModuleDeps(t *testing.T) {
+	tables := []struct {
+		in       string
+		expected []string
+	}{
+		{"nls-iso8859-1", []string{"kernel/fs/nls/nls_iso8859-1.ko.xz"}},
+		{"gpu_sched", []string{"kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz"}},
+		{"dw-wdt", []string{"kernel/drivers/watchdog/dw_wdt.ko.xz",
+			"kernel/drivers/watchdog/watchdog.ko.xz"}},
+		{"gl518sm", []string{"kernel/drivers/hwmon/gl518sm.ko.xz"}},
+		{"msm", []string{"kernel/drivers/gpu/drm/msm/msm.ko",
+			"kernel/drivers/gpu/drm/drm_kms_helper.ko"}},
+	}
+	for _, table := range tables {
+		out, err := getModuleDeps(table.in, strings.NewReader(testModuleDep))
+		if err != nil {
+			t.Errorf("unexpected error with input: %q, error: %q", table.expected, err)
+		}
+		if !stringSlicesEqual(out, table.expected) {
+			t.Errorf("Expected: %q, got: %q", table.expected, out)
+		}
+	}
+}

pkgs/archive/archive.go

@@ -1,26 +1,28 @@
 // Copyright 2021 Clayton Craft <clayton@craftyguy.net>
 // SPDX-License-Identifier: GPL-3.0-or-later
+
 package archive
 
 import (
 	"bytes"
 	"compress/flate"
-	"crypto/sha256"
+	"fmt"
-	"encoding/hex"
-	"github.com/cavaliercoder/go-cpio"
-	"github.com/klauspost/pgzip"
-	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/misc"
 	"io"
 	"log"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"sort"
 	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/cavaliercoder/go-cpio"
+	"github.com/klauspost/pgzip"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/misc"
 )
 
 type Archive struct {
-	Dirs       misc.StringSet
+	items      archiveItems
-	Files      misc.StringSet
 	cpioWriter *cpio.Writer
 	buf        *bytes.Buffer
 }
@@ -29,166 +31,184 @@ func New() (*Archive, error) {
 	buf := new(bytes.Buffer)
 	archive := &Archive{
 		cpioWriter: cpio.NewWriter(buf),
-		Files:      make(misc.StringSet),
-		Dirs:       make(misc.StringSet),
 		buf:        buf,
 	}
 
 	return archive, nil
 }
 
+type archiveItem struct {
+	sourcePath string
+	header     *cpio.Header
+}
+
+type archiveItems struct {
+	items []archiveItem
+	sync.RWMutex
+}
+
+// Adds the given item to the archiveItems, only if it doesn't already exist in
+// the list. The items are kept sorted in ascending order.
+func (a *archiveItems) Add(item archiveItem) {
+	a.Lock()
+	defer a.Unlock()
+
+	if len(a.items) < 1 {
+		// empty list
+		a.items = append(a.items, item)
+		return
+	}
+
+	// find existing item, or index of where new item should go
+	i := sort.Search(len(a.items), func(i int) bool {
+		return strings.Compare(item.header.Name, a.items[i].header.Name) <= 0
+	})
+
+	if i >= len(a.items) {
+		// doesn't exist in list, but would be at the very end
+		a.items = append(a.items, item)
+		return
+	}
+
+	if strings.Compare(a.items[i].header.Name, item.header.Name) == 0 {
+		// already in list
+		return
+	}
+
+	// grow list by 1, shift right at index, and insert new string at index
+	a.items = append(a.items, archiveItem{})
+	copy(a.items[i+1:], a.items[i:])
+	a.items[i] = item
+}
+
+// iterate through items and send each one over the returned channel
+func (a *archiveItems) IterItems() <-chan archiveItem {
+	ch := make(chan archiveItem)
+	go func() {
+		a.RLock()
+		defer a.RUnlock()
+
+		for _, item := range a.items {
+			ch <- item
+		}
+		close(ch)
+	}()
+	return ch
+}
+
 func (archive *Archive) Write(path string, mode os.FileMode) error {
 	if err := archive.writeCpio(); err != nil {
 		return err
 	}
 
 	if err := archive.cpioWriter.Close(); err != nil {
-		return err
+		return fmt.Errorf("archive.Write: error closing archive: %w", err)
 	}
 
 	// Write archive to path
 	if err := archive.writeCompressed(path, mode); err != nil {
-		log.Print("Unable to write archive to location: ", path)
+		return fmt.Errorf("unable to write archive to location %q: %w", path, err)
-		return err
-	}
-
-	// test the archive to make sure it's valid
-	if err := test(path); err != nil {
-		log.Print("Verification of archive failed!")
-		return err
 	}
 
 	if err := os.Chmod(path, mode); err != nil {
-		return err
+		return fmt.Errorf("unable to chmod %q to %s: %w", path, mode, err)
 	}
 
 	return nil
 }
 
-func checksum(path string) (string, error) {
+// Adds the given items in the map to the archive. The map format is {source path:dest path}.
-	var sum string
+// Internally this just calls AddItem on each key,value pair in the map.
-
+func (archive *Archive) AddItems(paths map[string]string) error {
-	buf := make([]byte, 64*1024)
+	for s, d := range paths {
-	sha256 := sha256.New()
+		if err := archive.AddItem(s, d); err != nil {
-	fd, err := os.Open(path)
+			return err
-	defer fd.Close()
-
-	if err != nil {
-		log.Print("Unable to checksum: ", path)
-		return sum, err
-	}
-
-	// Read file in chunks
-	for {
-		bytes, err := fd.Read(buf)
-		if bytes > 0 {
-			_, err := sha256.Write(buf[:bytes])
-			if err != nil {
-				log.Print("Unable to checksum: ", path)
-				return sum, err
-			}
-		}
-
-		if err == io.EOF {
-			break
 		}
 	}
-	sum = hex.EncodeToString(sha256.Sum(nil))
+	return nil
-	return sum, nil
 }
 
-func (archive *Archive) AddFile(file string, dest string) error {
+// Adds the given file or directory at "source" to the archive at "dest"
+func (archive *Archive) AddItem(source string, dest string) error {
+
+	sourceStat, err := os.Lstat(source)
+	if err != nil {
+		e, ok := err.(*os.PathError)
+		if e.Err == syscall.ENOENT && ok {
+			// doesn't exist in current filesystem, assume it's a new directory
+			return archive.addDir(dest)
+		}
+		return fmt.Errorf("AddItem: failed to get stat for %q: %w", source, err)
+	}
+
+	if sourceStat.Mode()&os.ModeDir != 0 {
+		return archive.addDir(dest)
+	}
+
+	return archive.addFile(source, dest)
+}
+
+func (archive *Archive) addFile(source string, dest string) error {
 	if err := archive.addDir(filepath.Dir(dest)); err != nil {
 		return err
 	}
 
-	if archive.Files[file] {
+	sourceStat, err := os.Lstat(source)
-		// Already written to cpio
-		return nil
-	}
-
-	fileStat, err := os.Lstat(file)
 	if err != nil {
-		log.Print("AddFile: failed to stat file: ", file)
+		log.Print("addFile: failed to stat file: ", source)
 		return err
 	}
 
 	// Symlink: write symlink to archive then set 'file' to link target
-	if fileStat.Mode()&os.ModeSymlink != 0 {
+	if sourceStat.Mode()&os.ModeSymlink != 0 {
 		// log.Printf("File %q is a symlink", file)
-		target, err := os.Readlink(file)
+		target, err := os.Readlink(source)
 		if err != nil {
-			log.Print("AddFile: failed to get symlink target: ", file)
+			log.Print("addFile: failed to get symlink target: ", source)
 			return err
 		}
 
 		destFilename := strings.TrimPrefix(dest, "/")
-		hdr := &cpio.Header{
-			Name:     destFilename,
-			Linkname: target,
-			Mode:     0644 | cpio.ModeSymlink,
-			Size:     int64(len(target)),
-			// Checksum: 1,
-		}
-		if err := archive.cpioWriter.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err = archive.cpioWriter.Write([]byte(target)); err != nil {
-			return err
-		}
 
-		archive.Files[file] = true
+		archive.items.Add(archiveItem{
+			sourcePath: source,
+			header: &cpio.Header{
+				Name:     destFilename,
+				Linkname: target,
+				Mode:     0644 | cpio.ModeSymlink,
+				Size:     int64(len(target)),
+				// Checksum: 1,
+			},
+		})
+
 		if filepath.Dir(target) == "." {
-			target = filepath.Join(filepath.Dir(file), target)
+			target = filepath.Join(filepath.Dir(source), target)
 		}
 		// make sure target is an absolute path
 		if !filepath.IsAbs(target) {
-			target, err = misc.RelativeSymlinkTargetToDir(target, filepath.Dir(file))
+			target, err = misc.RelativeSymlinkTargetToDir(target, filepath.Dir(source))
+			if err != nil {
+				return err
+			}
 		}
 		// TODO: add verbose mode, print stuff like this:
 		// log.Printf("symlink: %q, target: %q", file, target)
 		// write symlink target
-		err = archive.AddFile(target, target)
+		err = archive.addFile(target, target)
 		return err
 	}
 
-	// log.Printf("writing file: %q", file)
-
-	fd, err := os.Open(file)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-
 	destFilename := strings.TrimPrefix(dest, "/")
-	hdr := &cpio.Header{
-		Name: destFilename,
-		Mode: cpio.FileMode(fileStat.Mode().Perm()),
-		Size: fileStat.Size(),
-		// Checksum: 1,
-	}
-	if err := archive.cpioWriter.WriteHeader(hdr); err != nil {
-		return err
-	}
 
-	if _, err = io.Copy(archive.cpioWriter, fd); err != nil {
+	archive.items.Add(archiveItem{
-		return err
+		sourcePath: source,
-	}
+		header: &cpio.Header{
-
+			Name: destFilename,
-	archive.Files[file] = true
+			Mode: cpio.FileMode(sourceStat.Mode().Perm()),
-
+			Size: sourceStat.Size(),
-	return nil
+			// Checksum: 1,
-}
+		},
-
+	})
-// Use busybox gzip to test archive
-func test(path string) error {
-	cmd := exec.Command("busybox", "gzip", "-t", path)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		log.Print("'boot-deploy' command failed: ")
-		return err
-	}
 
 	return nil
 }
@@ -226,29 +246,48 @@ func (archive *Archive) writeCompressed(path string, mode os.FileMode) error {
 }
 
 func (archive *Archive) writeCpio() error {
-	// Write any dirs added explicitly
+	// having a transient function for actually adding files to the archive
-	for dir := range archive.Dirs {
+	// allows the deferred fd.close to run after every copy and prevent having
-		archive.addDir(dir)
+	// tons of open file handles until the copying is all done
+	copyToArchive := func(source string, header *cpio.Header) error {
+
+		if err := archive.cpioWriter.WriteHeader(header); err != nil {
+			return fmt.Errorf("archive.writeCpio: unable to write header: %w", err)
+		}
+
+		// don't copy actual dirs into the archive, writing the header is enough
+		if !header.Mode.IsDir() {
+			if header.Mode.IsRegular() {
+				fd, err := os.Open(source)
+				if err != nil {
+					return fmt.Errorf("archive.writeCpio: uname to open file %q, %w", source, err)
+				}
+				defer fd.Close()
+				if _, err := io.Copy(archive.cpioWriter, fd); err != nil {
+					return fmt.Errorf("archive.writeCpio: unable to write out archive: %w", err)
+				}
+			} else if header.Linkname != "" {
+				// the contents of a symlink is just need the link name
+				if _, err := archive.cpioWriter.Write([]byte(header.Linkname)); err != nil {
+					return fmt.Errorf("archive.writeCpio: unable to write out symlink: %w", err)
+				}
+			} else {
+				return fmt.Errorf("archive.writeCpio: unknown type for file: %s", source)
+			}
+		}
+
+		return nil
 	}
 
-	// Write files and any missing parent dirs
+	for i := range archive.items.IterItems() {
-	for file, imported := range archive.Files {
+		if err := copyToArchive(i.sourcePath, i.header); err != nil {
-		if imported {
-			continue
-		}
-		if err := archive.AddFile(file, file); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
 
 func (archive *Archive) addDir(dir string) error {
-	if archive.Dirs[dir] {
-		// Already imported
-		return nil
-	}
 	if dir == "/" {
 		dir = "."
 	}
@@ -256,19 +295,13 @@ func (archive *Archive) addDir(dir string) error {
 	subdirs := strings.Split(strings.TrimPrefix(dir, "/"), "/")
 	for i, subdir := range subdirs {
 		path := filepath.Join(strings.Join(subdirs[:i], "/"), subdir)
-		if archive.Dirs[path] {
+		archive.items.Add(archiveItem{
-			// Subdir already imported
+			sourcePath: path,
-			continue
+			header: &cpio.Header{
-		}
+				Name: path,
-		err := archive.cpioWriter.WriteHeader(&cpio.Header{
+				Mode: cpio.ModeDir | 0755,
-			Name: path,
+			},
-			Mode: cpio.ModeDir | 0755,
 		})
-		if err != nil {
-			return err
-		}
-		archive.Dirs[path] = true
-		// log.Print("wrote dir: ", path)
 	}
 
 	return nil

pkgs/archive/archive_test.go

@@ -0,0 +1,189 @@
+// Copyright 2022 Clayton Craft <clayton@craftyguy.net>
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package archive
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cavaliercoder/go-cpio"
+)
+
+func TestArchiveItemsAdd(t *testing.T) {
+	subtests := []struct {
+		name     string
+		inItems  []archiveItem
+		inItem   archiveItem
+		expected []archiveItem
+	}{
+		{
+			name:    "empty list",
+			inItems: []archiveItem{},
+			inItem: archiveItem{
+				sourcePath: "/foo/bar",
+				header:     &cpio.Header{Name: "/foo/bar"},
+			},
+			expected: []archiveItem{
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+			},
+		},
+		{
+			name: "already exists",
+			inItems: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+			},
+			inItem: archiveItem{
+				sourcePath: "/foo",
+				header:     &cpio.Header{Name: "/foo"},
+			},
+			expected: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+			},
+		},
+		{
+			name: "add new",
+			inItems: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+				{
+					sourcePath: "/foo/bar1",
+					header:     &cpio.Header{Name: "/foo/bar1"},
+				},
+			},
+			inItem: archiveItem{
+				sourcePath: "/foo/bar0",
+				header:     &cpio.Header{Name: "/foo/bar0"},
+			},
+			expected: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+				{
+					sourcePath: "/foo/bar0",
+					header:     &cpio.Header{Name: "/foo/bar0"},
+				},
+				{
+					sourcePath: "/foo/bar1",
+					header:     &cpio.Header{Name: "/foo/bar1"},
+				},
+			},
+		},
+		{
+			name: "add new at beginning",
+			inItems: []archiveItem{
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+			},
+			inItem: archiveItem{
+				sourcePath: "/bazz/bar",
+				header:     &cpio.Header{Name: "/bazz/bar"},
+			},
+			expected: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/foo/bar",
+					header:     &cpio.Header{Name: "/foo/bar"},
+				},
+			},
+		},
+		{
+			name: "add new at end",
+			inItems: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+			},
+			inItem: archiveItem{
+				sourcePath: "/zzz/bazz",
+				header:     &cpio.Header{Name: "/zzz/bazz"},
+			},
+			expected: []archiveItem{
+				{
+					sourcePath: "/bazz/bar",
+					header:     &cpio.Header{Name: "/bazz/bar"},
+				},
+				{
+					sourcePath: "/foo",
+					header:     &cpio.Header{Name: "/foo"},
+				},
+				{
+					sourcePath: "/zzz/bazz",
+					header:     &cpio.Header{Name: "/zzz/bazz"},
+				},
+			},
+		},
+	}
+
+	for _, st := range subtests {
+		t.Run(st.name, func(t *testing.T) {
+			a := archiveItems{items: st.inItems}
+			a.Add(st.inItem)
+			if !reflect.DeepEqual(st.expected, a.items) {
+				t.Fatal("expected:", st.expected, " got: ", a.items)
+			}
+		})
+	}
+}

pkgs/deviceinfo/deviceinfo.go

@@ -1,53 +1,130 @@
 // Copyright 2021 Clayton Craft <clayton@craftyguy.net>
 // SPDX-License-Identifier: GPL-3.0-or-later
+
 package deviceinfo
 
 import (
-	"errors"
+	"bufio"
-	"github.com/BurntSushi/toml"
+	"fmt"
+	"io"
+	"log"
 	"os"
+	"reflect"
+	"strings"
 )
 
-// Note: fields must be exported (start with capital letter)
-// https://github.com/BurntSushi/toml/issues/121
 type DeviceInfo struct {
-	Deviceinfo_append_dtb                      string
+	AppendDtb                     string
-	Deviceinfo_arch                            string
+	Arch                          string
-	Deviceinfo_bootimg_append_seandroidenforce string
+	UbootBoardname                string
-	Deviceinfo_bootimg_blobpack                string
+	BootimgAppendSEAndroidEnforce string
-	Deviceinfo_bootimg_dtb_second              string
+	BootimgBlobpack               string
-	Deviceinfo_bootimg_mtk_mkimage             string
+	BootimgDtbSecond              string
-	Deviceinfo_bootimg_pxa                     string
+	BootimgMtkMkimage             string
-	Deviceinfo_bootimg_qcdt                    string
+	BootimgPxa                    string
-	Deviceinfo_dtb                             string
+	BootimgQcdt                   string
-	Deviceinfo_flash_offset_base               string
+	Dtb                           string
-	Deviceinfo_flash_offset_kernel             string
+	FlashKernelOnUpdate           string
-	Deviceinfo_flash_offset_ramdisk            string
+	FlashOffsetBase               string
-	Deviceinfo_flash_offset_second             string
+	FlashOffsetKernel             string
-	Deviceinfo_flash_offset_tags               string
+	FlashOffsetRamdisk            string
-	Deviceinfo_flash_pagesize                  string
+	FlashOffsetSecond             string
-	Deviceinfo_generate_bootimg                string
+	FlashOffsetTags               string
-	Deviceinfo_generate_legacy_uboot_initfs    string
+	FlashPagesize                 string
-	Deviceinfo_mesa_driver                     string
+	GenerateBootimg               string
-	Deviceinfo_mkinitfs_postprocess            string
+	GenerateLegacyUbootInitfs     string
-	Deviceinfo_initfs_compression              string
+	InitfsCompression             string
-	Deviceinfo_kernel_cmdline                  string
+	KernelCmdline                 string
-	Deviceinfo_legacy_uboot_load_address       string
+	LegacyUbootLoadAddress        string
-	Deviceinfo_modules_initfs                  string
+	MesaDriver                    string
-	Deviceinfo_flash_kernel_on_update          string
+	MkinitfsPostprocess           string
+	ModulesInitfs                 string
 }
 
-func ReadDeviceinfo() (DeviceInfo, error) {
+func ReadDeviceinfo(file string) (DeviceInfo, error) {
-	file := "/etc/deviceinfo"
 	var deviceinfo DeviceInfo
 
-	_, err := os.Stat(file)
+	fd, err := os.Open(file)
 	if err != nil {
-		return deviceinfo, errors.New("Unable to find deviceinfo: " + file)
-	}
-
-	if _, err := toml.DecodeFile(file, &deviceinfo); err != nil {
 		return deviceinfo, err
 	}
+	defer fd.Close()
+
+	if err := unmarshal(fd, &deviceinfo); err != nil {
+		return deviceinfo, err
+	}
+
 	return deviceinfo, nil
 }
+
+// Unmarshals a deviceinfo into a DeviceInfo struct
+func unmarshal(r io.Reader, devinfo *DeviceInfo) error {
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		line := s.Text()
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		// line isn't setting anything, so just ignore it
+		if !strings.Contains(line, "=") {
+			continue
+		}
+
+		// sometimes line has a comment at the end after setting an option
+		line = strings.SplitN(line, "#", 2)[0]
+		line = strings.TrimSpace(line)
+
+		// must support having '=' in the value (e.g. kernel cmdline)
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
+		}
+
+		name, val := parts[0], parts[1]
+		val = strings.ReplaceAll(val, "\"", "")
+
+		if name == "deviceinfo_format_version" && val != "0" {
+			return fmt.Errorf("deviceinfo format version %q is not supported", val)
+		}
+
+		fieldName := nameToField(name)
+
+		if fieldName == "" {
+			return fmt.Errorf("error parsing deviceinfo line, invalid format: %s", line)
+		}
+
+		field := reflect.ValueOf(devinfo).Elem().FieldByName(fieldName)
+		if !field.IsValid() {
+			// an option that meets the deviceinfo "specification", but isn't
+			// one we care about in this module
+			continue
+		}
+		field.SetString(val)
+	}
+	if err := s.Err(); err != nil {
+		log.Print("unable to parse deviceinfo: ", err)
+		return err
+	}
+
+	return nil
+}
+
+// Convert string into the string format used for DeviceInfo fields.
+// Note: does not test that the resulting field name is a valid field in the
+// DeviceInfo struct!
+func nameToField(name string) string {
+	var field string
+	parts := strings.Split(name, "_")
+	for _, p := range parts {
+		if p == "deviceinfo" {
+			continue
+		}
+		if len(p) < 1 {
+			continue
+		}
+		field = field + strings.ToUpper(p[:1]) + p[1:]
+	}
+
+	return field
+}

pkgs/deviceinfo/deviceinfo_test.go

@@ -0,0 +1,81 @@
+// Copyright 2021 Clayton Craft <clayton@craftyguy.net>
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package deviceinfo
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Test conversion of name to DeviceInfo struct field format
+func TestNameToField(t *testing.T) {
+	tables := []struct {
+		in       string
+		expected string
+	}{
+		{"deviceinfo_dtb", "Dtb"},
+		{"dtb", "Dtb"},
+		{"deviceinfo_modules_initfs", "ModulesInitfs"},
+		{"modules_initfs", "ModulesInitfs"},
+		{"deviceinfo_modules_initfs___", "ModulesInitfs"},
+	}
+
+	for _, table := range tables {
+		out := nameToField(table.in)
+		if out != table.expected {
+			t.Errorf("expected: %q, got: %q", table.expected, out)
+		}
+	}
+}
+
+// Test unmarshalling with lines in deviceinfo
+func TestUnmarshal(t *testing.T) {
+	tables := []struct {
+		// field is just used for reflection within the test, so it must be a
+		// valid DeviceInfo field
+		field    string
+		in       string
+		expected string
+	}{
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"\n", "panfrost foo bar bazz"},
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"panfrost foo bar bazz\"", "panfrost foo bar bazz"},
+		// line with multiple '='
+		{"KernelCmdline",
+			"deviceinfo_kernel_cmdline=\"PMOS_NO_OUTPUT_REDIRECT fw_devlink=off nvme_core.default_ps_max_latency_us=5500 pcie_aspm.policy=performance\"\n",
+			"PMOS_NO_OUTPUT_REDIRECT fw_devlink=off nvme_core.default_ps_max_latency_us=5500 pcie_aspm.policy=performance"},
+		// empty option
+		{"ModulesInitfs", "deviceinfo_modules_initfs=\"\"\n", ""},
+		{"Dtb", "deviceinfo_dtb=\"freescale/imx8mq-librem5-r2 freescale/imx8mq-librem5-r3 freescale/imx8mq-librem5-r4\"\n",
+			"freescale/imx8mq-librem5-r2 freescale/imx8mq-librem5-r3 freescale/imx8mq-librem5-r4"},
+		// valid deviceinfo line, just not used in this module
+		{"", "deviceinfo_codename=\"pine64-pinebookpro\"", ""},
+		// line with comment at the end
+		{"MesaDriver", "deviceinfo_mesa_driver=\"panfrost\"  # this is a nice driver", "panfrost"},
+		{"", "# this is a comment!\n", ""},
+		// empty lines are fine
+		{"", "", ""},
+		// line with whitepace characters only
+		{"", " \t \n\r", ""},
+	}
+	var d DeviceInfo
+	for _, table := range tables {
+		testName := fmt.Sprintf("unmarshal::'%s':", strings.ReplaceAll(table.in, "\n", "\\n"))
+		if err := unmarshal(strings.NewReader(table.in), &d); err != nil {
+			t.Errorf("%s received an unexpected err: ", err)
+		}
+
+		// Check against expected value
+		field := reflect.ValueOf(&d).Elem().FieldByName(table.field)
+		out := ""
+		if table.field != "" {
+			out = field.String()
+		}
+		if out != table.expected {
+			t.Errorf("%s expected: %q, got: %q", testName, table.expected, out)
+		}
+	}
+
+}

pkgs/misc/misc.go

@@ -1,5 +1,6 @@
-// Copyright 2021 Clayton Craft <clayton@craftyguy.net>
+// Copyright 2022 Clayton Craft <clayton@craftyguy.net>
 // SPDX-License-Identifier: GPL-3.0-or-later
+
 package misc
 
 import (
@@ -9,8 +10,6 @@ import (
 	"path/filepath"
 )
 
-type StringSet map[string]bool
-
 // Converts a relative symlink target path (e.g. ../../lib/foo.so), that is
 // absolute path
 func RelativeSymlinkTargetToDir(symPath string, dir string) (string, error) {
@@ -47,3 +46,31 @@ func FreeSpace(path string) (uint64, error) {
 	size := stat.Bavail * uint64(stat.Bsize)
 	return size, nil
 }
+
+// Merge the contents of "b" into "a", overwriting any previously existing keys
+// in "a"
+func Merge(a map[string]string, b map[string]string) {
+	for k, v := range b {
+		a[k] = v
+	}
+}
+
+// Removes duplicate entries from the given string slice and returns a slice
+// with the unique values
+func RemoveDuplicates(in []string) (out []string) {
+	// use a map to "remove" duplicates. the value in the map is totally
+	// irrelevant
+	outMap := make(map[string]bool)
+	for _, s := range in {
+		if ok := outMap[s]; !ok {
+			outMap[s] = true
+		}
+	}
+
+	out = make([]string, 0, len(outMap))
+	for k := range outMap {
+		out = append(out, k)
+	}
+
+	return
+}

pkgs/misc/misc_test.go

@@ -0,0 +1,125 @@
+// Copyright 2022 Clayton Craft <clayton@craftyguy.net>
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package misc
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestMerge(t *testing.T) {
+	subtests := []struct {
+		name     string
+		inA      map[string]string
+		inB      map[string]string
+		expected map[string]string
+	}{
+		{
+			name: "empty B",
+			inA: map[string]string{
+				"foo":    "bar",
+				"banana": "airplane",
+			},
+			inB: map[string]string{},
+			expected: map[string]string{
+				"foo":    "bar",
+				"banana": "airplane",
+			},
+		},
+		{
+			name: "empty A",
+			inA:  map[string]string{},
+			inB: map[string]string{
+				"foo":    "bar",
+				"banana": "airplane",
+			},
+			expected: map[string]string{
+				"foo":    "bar",
+				"banana": "airplane",
+			},
+		},
+		{
+			name: "both populated, some duplicates",
+			inA: map[string]string{
+				"bar":    "bazz",
+				"banana": "yellow",
+				"guava":  "green",
+			},
+			inB: map[string]string{
+				"foo":    "bar",
+				"banana": "airplane",
+			},
+			expected: map[string]string{
+				"foo":    "bar",
+				"guava":  "green",
+				"banana": "airplane",
+				"bar":    "bazz",
+			},
+		},
+	}
+
+	for _, st := range subtests {
+		t.Run(st.name, func(t *testing.T) {
+			out := st.inA
+			Merge(out, st.inB)
+			if !reflect.DeepEqual(st.expected, out) {
+				t.Fatalf("expected: %q, got: %q\n", st.expected, out)
+			}
+		})
+	}
+}
+
+func TestRemoveDuplicates(t *testing.T) {
+	subtests := []struct {
+		name     string
+		in       []string
+		expected []string
+	}{
+		{
+			name: "no duplicates",
+			in: []string{
+				"foo",
+				"bar",
+				"banana",
+				"airplane",
+			},
+			expected: []string{
+				"foo",
+				"bar",
+				"banana",
+				"airplane",
+			},
+		},
+		{
+			name: "all duplicates",
+			in: []string{
+				"foo",
+				"foo",
+				"foo",
+				"foo",
+			},
+			expected: []string{
+				"foo",
+			},
+		},
+		{
+			name:     "empty",
+			in:       []string{},
+			expected: []string{},
+		},
+	}
+
+	for _, st := range subtests {
+		t.Run(st.name, func(t *testing.T) {
+			// note: sorting to make comparison easier later
+			sort.Strings(st.expected)
+			out := RemoveDuplicates(st.in)
+			sort.Strings(out)
+			if !reflect.DeepEqual(st.expected, out) {
+				t.Fatalf("expected: %q, got: %q\n", st.expected, out)
+			}
+		})
+	}
+}