cmd/mkinitfs: use filelist/modules

cmd/mkinitfs/main.go

@@ -14,12 +14,12 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"time"
 
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/archive"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/osksdl"
+	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/filelist/modules"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/internal/misc"
 	"gitlab.com/postmarketOS/postmarketos-mkinitfs/pkgs/deviceinfo"
 )
@@ -431,84 +431,6 @@ func getInitfsFiles(devinfo deviceinfo.DeviceInfo) (files []string, err error) {
 	return
 }
 
-func getInitfsModules(devinfo deviceinfo.DeviceInfo, kernelVer string) (files []string, err error) {
-	log.Println("- Including kernel modules")
-
-	modDir := filepath.Join("/lib/modules", kernelVer)
-	if !misc.Exists(modDir) {
-		// dir /lib/modules/<kernel> if kernel built without module support, so just print a message
-		log.Printf("-- kernel module directory not found: %q, not including modules", modDir)
-		return
-	}
-
-	// modules.* required by modprobe
-	modprobeFiles, _ := filepath.Glob(filepath.Join(modDir, "modules.*"))
-	files = append(files, modprobeFiles...)
-
-	// module name (without extension), or directory (trailing slash is important! globs OK)
-	requiredModules := []string{
-		"loop",
-		"dm-crypt",
-		"kernel/fs/overlayfs/",
-		"kernel/crypto/",
-		"kernel/arch/*/crypto/",
-	}
-
-	for _, item := range requiredModules {
-		dir, file := filepath.Split(item)
-		if file == "" {
-			// item is a directory
-			dir = filepath.Join(modDir, dir)
-			dirs, _ := filepath.Glob(dir)
-			for _, d := range dirs {
-				if filelist, err := getModulesInDir(d); err != nil {
-					return nil, fmt.Errorf("getInitfsModules: unable to get modules dir %q: %w", d, err)
-				} else {
-					files = append(files, filelist...)
-				}
-			}
-		} else if dir == "" {
-			// item is a module name
-			if filelist, err := getModule(file, modDir); err != nil {
-				return nil, fmt.Errorf("getInitfsModules: unable to get module %q: %w", file, err)
-			} else {
-				files = append(files, filelist...)
-			}
-		} else {
-			log.Printf("Unknown module entry: %q", item)
-		}
-	}
-
-	// deviceinfo modules
-	for _, module := range strings.Fields(devinfo.ModulesInitfs) {
-		if filelist, err := getModule(module, modDir); err != nil {
-			return nil, fmt.Errorf("getInitfsModules: unable to get modules from deviceinfo: %w", err)
-		} else {
-			files = append(files, filelist...)
-		}
-	}
-
-	// /etc/postmarketos-mkinitfs/modules/*.modules
-	initfsModFiles, _ := filepath.Glob("/etc/postmarketos-mkinitfs/modules/*.modules")
-	for _, modFile := range initfsModFiles {
-		f, err := os.Open(modFile)
-		if err != nil {
-			return nil, fmt.Errorf("getInitfsModules: unable to open mkinitfs modules file %q: %w", modFile, err)
-		}
-		defer f.Close()
-		s := bufio.NewScanner(f)
-		for s.Scan() {
-			if filelist, err := getModule(s.Text(), modDir); err != nil {
-				return nil, fmt.Errorf("getInitfsModules: unable to get module file %q: %w", s.Text(), err)
-			} else {
-				files = append(files, filelist...)
-			}
-		}
-	}
-
-	return
-}
-
 func getKernelReleaseFile() (string, error) {
 	files, _ := filepath.Glob("/usr/share/kernel/*/kernel.release")
 	// only one kernel flavor supported
@@ -609,13 +531,15 @@ func generateInitfs(name string, path string, kernVer string, devinfo deviceinfo
 		}
 	}
 
-	if files, err := getInitfsModules(devinfo, kernVer); err != nil {
+	log.Println("- Including kernel modules")
+	modules := modules.New(strings.Fields(devinfo.ModulesInitfs))
+	if list, err := modules.List(); err != nil {
 		return err
 	} else {
 		items := make(map[string]string)
 		// copy files into a map, where the source(key) and dest(value) are the
 		// same
-		for _, f := range files {
+		for _, f := range list {
 			items[f] = f
 		}
 		if err := initfsArchive.AddItems(items); err != nil {
@@ -678,89 +602,3 @@ func generateInitfsExtra(name string, path string, devinfo deviceinfo.DeviceInfo
 
 	return nil
 }
-
-func stripExts(file string) string {
-	return strings.Split(file, ".")[0]
-}
-
-func getModulesInDir(modPath string) (files []string, err error) {
-	err = filepath.Walk(modPath, func(path string, f os.FileInfo, err error) error {
-		// TODO: need to support more extensions?
-		if filepath.Ext(path) != ".ko" && filepath.Ext(path) != ".xz" {
-			return nil
-		}
-		files = append(files, path)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return
-}
-
-// Given a module name, e.g. 'dwc_wdt', resolve the full path to the module
-// file and all of its dependencies.
-// Note: it's not necessarily fatal if the module is not found, since it may
-// have been built into the kernel
-// TODO: look for it in modules.builtin, and make it fatal if it can't be found
-// anywhere
-func getModule(modName string, modDir string) (files []string, err error) {
-
-	modDep := filepath.Join(modDir, "modules.dep")
-	if !misc.Exists(modDep) {
-		return nil, fmt.Errorf("kernel module.dep not found: %s", modDir)
-	}
-
-	fd, err := os.Open(modDep)
-	if err != nil {
-		return nil, fmt.Errorf("unable to open modules.dep: %w", err)
-	}
-	defer fd.Close()
-
-	deps, err := getModuleDeps(modName, fd)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, dep := range deps {
-		p := filepath.Join(modDir, dep)
-		if !misc.Exists(p) {
-			return nil, fmt.Errorf("tried to include a module that doesn't exist in the modules directory (%s): %s", modDir, p)
-		}
-		files = append(files, p)
-	}
-
-	return
-}
-
-// Get the canonicalized name for the module as represented in the given modules.dep io.reader
-func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
-	var deps []string
-
-	// split the module name on - and/or _, build a regex for matching
-	splitRe := regexp.MustCompile("[-_]+")
-	modNameReStr := splitRe.ReplaceAllString(modName, "[-_]+")
-	re := regexp.MustCompile("^" + modNameReStr + "$")
-
-	s := bufio.NewScanner(modulesDep)
-	for s.Scan() {
-		fields := strings.Fields(s.Text())
-		if len(fields) == 0 {
-			continue
-		}
-		fields[0] = strings.TrimSuffix(fields[0], ":")
-
-		found := re.FindAll([]byte(filepath.Base(stripExts(fields[0]))), -1)
-		if len(found) > 0 {
-			deps = append(deps, fields...)
-			break
-		}
-	}
-	if err := s.Err(); err != nil {
-		log.Print("Unable to get module + dependencies: ", modName)
-		return deps, err
-	}
-
-	return deps, nil
-}

cmd/mkinitfs/main_test.go

@@ -1,82 +0,0 @@
-// Copyright 2021 Clayton Craft <clayton@craftyguy.net>
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-package main
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestStripExts(t *testing.T) {
-	tables := []struct {
-		in       string
-		expected string
-	}{
-		{"/foo/bar/bazz.tar", "/foo/bar/bazz"},
-		{"file.tar.gz.xz.zip", "file"},
-		{"another_file", "another_file"},
-		{"a.b.c.d.e.f.g.h.i", "a"},
-		{"virtio_blk.ko", "virtio_blk"},
-	}
-	for _, table := range tables {
-		out := stripExts(table.in)
-		if out != table.expected {
-			t.Errorf("Expected: %q, got: %q", table.expected, out)
-		}
-	}
-}
-
-func stringSlicesEqual(a []string, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i, v := range a {
-		if v != b[i] {
-			return false
-		}
-	}
-	return true
-}
-
-var testModuleDep string = `
-kernel/sound/soc/codecs/snd-soc-msm8916-digital.ko:
-kernel/net/sched/act_ipt.ko.xz: kernel/net/netfilter/x_tables.ko.xz
-kernel/drivers/watchdog/watchdog.ko.xz:
-kernel/drivers/usb/serial/ir-usb.ko.xz: kernel/drivers/usb/serial/usbserial.ko.xz
-kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz:
-kernel/drivers/hid/hid-alps.ko.xz:
-kernel/net/netfilter/xt_u32.ko.xz: kernel/net/netfilter/x_tables.ko.xz
-kernel/net/netfilter/xt_sctp.ko.xz: kernel/net/netfilter/x_tables.ko.xz
-kernel/drivers/hwmon/gl518sm.ko.xz:
-kernel/drivers/watchdog/dw_wdt.ko.xz: kernel/drivers/watchdog/watchdog.ko.xz
-kernel/net/bluetooth/hidp/hidp.ko.xz: kernel/net/bluetooth/bluetooth.ko.xz kernel/net/rfkill/rfkill.ko.xz kernel/crypto/ecdh_generic.ko.xz kernel/crypto/ecc.ko.xz
-kernel/fs/nls/nls_iso8859-1.ko.xz:
-kernel/net/vmw_vsock/vmw_vsock_virtio_transport.ko.xz: kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko.xz kernel/drivers/virtio/virtio.ko.xz kernel/drivers/virtio/virtio_ring.ko.xz kernel/net/vmw_vsock/vsock.ko.xz
-kernel/drivers/gpu/drm/panfrost/panfrost.ko.xz: kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz
-kernel/drivers/gpu/drm/msm/msm.ko: kernel/drivers/gpu/drm/drm_kms_helper.ko
-`
-
-func TestGetModuleDeps(t *testing.T) {
-	tables := []struct {
-		in       string
-		expected []string
-	}{
-		{"nls-iso8859-1", []string{"kernel/fs/nls/nls_iso8859-1.ko.xz"}},
-		{"gpu_sched", []string{"kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz"}},
-		{"dw-wdt", []string{"kernel/drivers/watchdog/dw_wdt.ko.xz",
-			"kernel/drivers/watchdog/watchdog.ko.xz"}},
-		{"gl518sm", []string{"kernel/drivers/hwmon/gl518sm.ko.xz"}},
-		{"msm", []string{"kernel/drivers/gpu/drm/msm/msm.ko",
-			"kernel/drivers/gpu/drm/drm_kms_helper.ko"}},
-	}
-	for _, table := range tables {
-		out, err := getModuleDeps(table.in, strings.NewReader(testModuleDep))
-		if err != nil {
-			t.Errorf("unexpected error with input: %q, error: %q", table.expected, err)
-		}
-		if !stringSlicesEqual(out, table.expected) {
-			t.Errorf("Expected: %q, got: %q", table.expected, out)
-		}
-	}
-}