get(Hook)Files: Use getFile, handle globs and dirs (MR 14)

This commit includes three changes:

1. Use getFile in getHookFiles to add the contents of file hooks
2. Within getFile, first attempted a glob expansion on the specified
   file expression. This allows specifying e.g.
   /lib/udev/rules.d/*.rules.
3. Within getFile, next if the path points to a directory, add all
   files including those from subdirectories. This allows specifying
   e.g. /usr/share/X11/xkb.

Relates to: postmarketOS/pmaports#1309

.gitlab-ci.yml

@@ -31,7 +31,7 @@ gofmt linting:
     - apk -q update --repository http://dl-4.alpinelinux.org/alpine/edge/testing
     - apk -q add --repository http://dl-4.alpinelinux.org/alpine/edge/testing go staticcheck
   script:
-    - .gitlab-ci/check_linting.sh
+    - .ci/check_linting.sh
 
 build:
   stage: build

main.go

@@ -86,18 +86,29 @@ func bootDeploy(workDir string, outDir string) error {
 	if len(kernels) == 0 {
 		return errors.New("Unable to find any kernels at " + filepath.Join(outDir, "vmlinuz*"))
 	}
-	kernFile, err := os.Open(kernels[0])
+
+	// Pick a kernel that does not have suffixes added by boot-deploy
+	var kernFile string
+	for _, f := range kernels {
+		if strings.HasSuffix(f, "-dtb") || strings.HasSuffix(f, "-mtk") {
+			continue
+		}
+		kernFile = f
+		break
+	}
+
+	kernFd, err := os.Open(kernFile)
 	if err != nil {
 		return err
 	}
-	defer kernFile.Close()
+	defer kernFd.Close()
 
 	kernFileCopy, err := os.Create(filepath.Join(workDir, "vmlinuz"))
 	if err != nil {
 		return err
 	}
 
-	if _, err = io.Copy(kernFileCopy, kernFile); err != nil {
+	if _, err = io.Copy(kernFileCopy, kernFd); err != nil {
 		return err
 	}
 	kernFileCopy.Close()
@@ -145,10 +156,9 @@ func getHookFiles(filesdir string) misc.StringSet {
 		defer f.Close()
 		s := bufio.NewScanner(f)
 		for s.Scan() {
-			if !exists(s.Text()) {
+			if err := getFile(files, s.Text(), true); err != nil {
 				log.Fatalf("Unable to find file %q required by %q", s.Text(), path)
 			}
-			files[s.Text()] = false
 		}
 		if err := s.Err(); err != nil {
 			log.Fatal(err)
@@ -232,13 +242,42 @@ func getFiles(files misc.StringSet, newFiles misc.StringSet, required bool) erro
 }
 
 func getFile(files misc.StringSet, file string, required bool) error {
-	if !exists(file) {
+	// Expand glob expression
+	expanded, _ := filepath.Glob(file)
+	if len(expanded) > 0 && expanded[0] != file {
+		for _, path := range expanded {
+			if err := getFile(files, path, required); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	fileInfo, err := os.Stat(file)
+	if err != nil {
 		if required {
 			return errors.New("getFile: File does not exist :" + file)
 		}
 		return nil
 	}
 
+	if fileInfo.IsDir() {
+		// Recurse over directory contents
+		err := filepath.Walk(file, func(path string, f os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if f.IsDir() {
+				return nil
+			}
+			return getFile(files, path, required)
+		})
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
 	files[file] = false
 
 	// get dependencies for binaries
@@ -247,8 +286,7 @@ func getFile(files misc.StringSet, file string, required bool) error {
 		return nil
 	}
 
-	err := getBinaryDeps(files, file)
+	if err := getBinaryDeps(files, file); err != nil {
-	if err != nil {
 		return err
 	}
 
@@ -403,6 +441,7 @@ func getInitfsFiles(files misc.StringSet, devinfo deviceinfo.DeviceInfo) error {
 		"/usr/sbin/telnetd":   false,
 		"/sbin/kpartx":        false,
 		"/etc/deviceinfo":     false,
+		"/usr/bin/unudhcpd":   false,
 	}
 
 	// Hook files & scripts
@@ -657,15 +696,8 @@ func getModuleDeps(modName string, modulesDep io.Reader) ([]string, error) {
 
 	// split the module name on - and/or _, build a regex for matching
 	splitRe := regexp.MustCompile("[-_]+")
-	var modNameReStr string
+	modNameReStr := splitRe.ReplaceAllString(modName, "[-_]+")
-	for _, s := range splitRe.Split(modName, -1) {
+	re := regexp.MustCompile("^" + modNameReStr + "$")
-		if modNameReStr != "" {
-			modNameReStr += "[-_]+" + s
-		} else {
-			modNameReStr = s
-		}
-	}
-	re := regexp.MustCompile(modNameReStr)
 
 	s := bufio.NewScanner(modulesDep)
 	for s.Scan() {

main_test.go

@@ -40,6 +40,7 @@ func stringSlicesEqual(a []string, b []string) bool {
 }
 
 var testModuleDep string = `
+kernel/sound/soc/codecs/snd-soc-msm8916-digital.ko:
 kernel/net/sched/act_ipt.ko.xz: kernel/net/netfilter/x_tables.ko.xz
 kernel/drivers/watchdog/watchdog.ko.xz:
 kernel/drivers/usb/serial/ir-usb.ko.xz: kernel/drivers/usb/serial/usbserial.ko.xz
@@ -53,6 +54,7 @@ kernel/net/bluetooth/hidp/hidp.ko.xz: kernel/net/bluetooth/bluetooth.ko.xz kerne
 kernel/fs/nls/nls_iso8859-1.ko.xz:
 kernel/net/vmw_vsock/vmw_vsock_virtio_transport.ko.xz: kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko.xz kernel/drivers/virtio/virtio.ko.xz kernel/drivers/virtio/virtio_ring.ko.xz kernel/net/vmw_vsock/vsock.ko.xz
 kernel/drivers/gpu/drm/panfrost/panfrost.ko.xz: kernel/drivers/gpu/drm/scheduler/gpu-sched.ko.xz
+kernel/drivers/gpu/drm/msm/msm.ko: kernel/drivers/gpu/drm/drm_kms_helper.ko
 `
 
 func TestGetModuleDeps(t *testing.T) {
@@ -65,6 +67,8 @@ func TestGetModuleDeps(t *testing.T) {
 		{"dw-wdt", []string{"kernel/drivers/watchdog/dw_wdt.ko.xz",
 			"kernel/drivers/watchdog/watchdog.ko.xz"}},
 		{"gl518sm", []string{"kernel/drivers/hwmon/gl518sm.ko.xz"}},
+		{"msm", []string{"kernel/drivers/gpu/drm/msm/msm.ko",
+			"kernel/drivers/gpu/drm/drm_kms_helper.ko"}},
 	}
 	for _, table := range tables {
 		out, err := getModuleDeps(table.in, strings.NewReader(testModuleDep))