firmware: copy only the necessary firmware(s) into initramfs

FIRMWARE=yes behavior is changed:
Only the minimum number of firmware files will be copied.
The list is generated using the `modinfo -F firmware [modules]...` command.
The ability to copy all firmware(s) is also available with a new
ALLFIRMWARE setting (see the modified genkernel.conf for more details).

As for changes in the source code:
gen_moddeps.sh: Significantly redesigned module list generation.
To get a list of modules, use the `mod_dep_list()` function instead
of `gen_dep_list()`. Modules that are not in the kernel (=n or invalid)
will be filtered out. Aliases will be replaced with real names
(including dependencies).

Signed-off-by: Dmitry Baranov <reagentoo@gmail.com>
Closes: https://github.com/gentoo/genkernel/pull/40
Signed-off-by: Sam James <sam@gentoo.org>

7 files changed, 130 insertions, 95 deletions

diff --git a/gen_cmdline.sh b/gen_cmdline.sh
index b6ad8612..5857d48a 100755
--- a/gen_cmdline.sh
+++ b/gen_cmdline.sh
@@ -62,6 +62,7 @@ longusage() {
   echo "	--ramdisk-modules	Copy required modules to the initramfs"
   echo "	--no-ramdisk-modules	Don't copy any modules to the initramfs"
   echo "	--all-ramdisk-modules	Copy all kernel modules to the initramfs"
+  echo "	--no-all-ramdisk-modules	Don't copy all kernel modules to the initramfs"
   echo "	--module-rebuild	Automatically run 'emerge @module-rebuild' when"
   echo "				necessary (and possible)"
   echo "	--no-module-rebuild	Don't automatically run 'emerge @module-rebuild'"
@@ -239,6 +240,9 @@ longusage() {
   echo "	--initramfs-symlink-name=<...>"
   echo "				Set initramfs symlink name"
   echo "	--firmware		Enable copying of firmware into initramfs"
+  echo "	--no-firmware	Disable copying of firmware into initramfs"
+  echo "	--all-firmware	Enable copying of all firmware into initramfs"
+  echo "	--no-all-firmware	Disable copying of all firmware into initramfs"
   echo "	--firmware-dir=<dir>"
   echo "				Specify directory to copy firmware from (defaults"
   echo "				to /lib/firmware)"
@@ -867,6 +871,10 @@ parse_cmdline() {
 			CMD_FIRMWARE=$(parse_optbool "$*")
 			print_info 3 "CMD_FIRMWARE: ${CMD_FIRMWARE}"
 			;;
+		--all-firmware|--no-all-firmware)
+			CMD_ALLFIRMWARE=$(parse_optbool "$*")
+			print_info 3 "CMD_FIRMWARE: ${CMD_ALLFIRMWARE}"
+			;;
 		--firmware-dir=*)
 			CMD_FIRMWARE_DIR="${*#*=}"
 			CMD_FIRMWARE="yes"
diff --git a/gen_configkernel.sh b/gen_configkernel.sh
index ebb3a44c..d074005b 100755
--- a/gen_configkernel.sh
+++ b/gen_configkernel.sh
@@ -347,13 +347,6 @@ config_kernel() {
 		unset kconfig_md5sum_old kconfig_md5sum_new
 	fi
 
-	# Check for suitable kmod
-	determine_KEXT
-	if ! isTrue "$(is_kext_supported_by_kmod "${KEXT}")"
-	then
-		gen_die "${KMOD_CMD} does not support chosen module compression algorithm. Please re-emerge sys-apps/kmod with USE=$(get_kext_kmod_use_flag "${KEXT}") enabled or adjust CONFIG_MODULE_COMPRESS_* kernel option!"
-	fi
-
 	local -a required_kernel_options
 	[ -f "${KCONFIG_MODIFIED_MARKER}" ] && rm "${KCONFIG_MODIFIED_MARKER}"
 
diff --git a/gen_determineargs.sh b/gen_determineargs.sh
index cefa614e..d29ad48a 100755
--- a/gen_determineargs.sh
+++ b/gen_determineargs.sh
@@ -445,6 +445,7 @@ determine_real_args() {
 	set_config_with_override BOOL   VIRTIO                                CMD_VIRTIO                                "no"
 	set_config_with_override BOOL   MULTIPATH                             CMD_MULTIPATH                             "no"
 	set_config_with_override BOOL   FIRMWARE                              CMD_FIRMWARE                              "no"
+	set_config_with_override BOOL   ALLFIRMWARE                           CMD_ALLFIRMWARE                           "no"
 	set_config_with_override STRING FIRMWARE_DIR                          CMD_FIRMWARE_DIR                          "/lib/firmware"
 	set_config_with_override STRING FIRMWARE_FILES                        CMD_FIRMWARE_FILES
 	set_config_with_override BOOL   FIRMWARE_INSTALL                      CMD_FIRMWARE_INSTALL                      "no"
diff --git a/gen_initramfs.sh b/gen_initramfs.sh
index ad61faa0..2cfa0fdb 100755
--- a/gen_initramfs.sh
+++ b/gen_initramfs.sh
@@ -1762,15 +1762,42 @@ append_firmware() {
 
 	mkdir -p "${TDIR}"/lib/firmware || gen_die "Failed to create '${TDIR}/lib/firmware'!"
 
-	if [ ${#FIRMWARE_FILES[@]} -gt 0 ]
+	local -a fwlist=()
+
+	if isTrue "${ALLFIRMWARE}"
 	then
-		pushd "${FIRMWARE_DIR}" &>/dev/null || gen_die "Failed to chdir to '${FIRMWARE_DIR}'!"
-		cp -rL --parents --target-directory="${TDIR}/lib/firmware" "${FIRMWARE_FILES[@]}" 2>/dev/null \
-			|| gen_die "Failed to copy firmware files (${FIRMWARE_FILES}) to '${TDIR}/lib/firmware'!"
-		popd &>/dev/null || gen_die "Failed to chdir!"
-	else
 		cp -a "${FIRMWARE_DIR}"/* "${TDIR}"/lib/firmware/ 2>/dev/null \
 			|| gen_die "Failed to copy firmware files to '${TDIR}/lib/firmware'!"
+	elif [ ${#FIRMWARE_FILES[@]} -gt 0 ]
+	then
+		fwlist=( "${FIRMWARE_FILES[@]}" )
+	else
+		local myfw=
+		while IFS= read -r -u 3 myfw
+		do
+			if [ -z "${myfw}" ]
+			then
+				gen_die "modinfo error!"
+			fi
+
+			if [ ! -f "${FIRMWARE_DIR}/${myfw}" ]
+			then
+				print_warning 3 "$(get_indent 3) - ${myfw} is missing; Ignoring ..."
+				continue
+			fi
+
+			fwlist+=( "${myfw}" )
+		done 3< <( (
+			modinfo -b "${KERNEL_MODULES_PREFIX%/}" -k "${KV}" -F firmware $(mod_dep_list) 2>/dev/null || echo
+		) | sort | uniq )
+	fi
+
+	if [ ${#fwlist[@]} -gt 0 ]
+	then
+		pushd "${FIRMWARE_DIR}" &>/dev/null || gen_die "Failed to chdir to '${FIRMWARE_DIR}'!"
+		cp -rL --parents --target-directory="${TDIR}/lib/firmware" "${fwlist[@]}" 2>/dev/null \
+			|| gen_die "Failed to copy firmware files to '${TDIR}/lib/firmware'!"
+		popd &>/dev/null || gen_die "Failed to chdir!"
 	fi
 
 	cd "${TDIR}" || gen_die "Failed to chdir to '${TDIR}'!"
@@ -1864,34 +1891,42 @@ append_modules() {
 		gen_die "${error_message}"
 	fi
 
-	determine_KEXT
-
 	cd "${modules_srcdir}" || gen_die "Failed to chdir to '${modules_srcdir}'!"
 
-	print_info 2 "$(get_indent 2)modules: >> Copying modules from '${modules_srcdir}' to initramfs ..."
+	print_info 2 "$(get_indent 2)modules: >> Searching modules in '${modules_srcdir}' ..."
 
-	local i= mymod=
-	local n_copied_modules=0
-	for i in $(gen_dep_list)
+	local mymod=
+	local -a modlist=()
+	while IFS= read -r -u 3 mymod
 	do
-		mymod=$(find . -name "${i}${KEXT}" 2>/dev/null | head -n 1)
 		if [ -z "${mymod}" ]
 		then
-			print_warning 3 "$(get_indent 3) - ${i}${KEXT} not found; Skipping ..."
-			continue;
+			gen_die "modinfo error!"
 		fi
 
-		print_info 3 "$(get_indent 3) - Copying ${i}${KEXT} ..."
-		cp -ax --parents --target-directory "${modules_dstdir}" "${mymod}" 2>/dev/null \
-			|| gen_die "Failed to copy '${modules_srcdir}/${mymod}' to '${modules_dstdir}'!"
-		n_copied_modules=$[$n_copied_modules+1]
-	done
+		if [ "${mymod}" = '(builtin)' ]
+		then
+			continue
+		fi
 
-	if [ ${n_copied_modules} -eq 0 ]
+		if [ ! -f "${mymod}" ]
+		then
+			gen_die "Module '${i}${KEXT}' is missing!"
+		fi
+
+		modlist+=( "${mymod/#${modules_srcdir}\//}" )
+	done 3< <( (
+		modinfo -b "${KERNEL_MODULES_PREFIX%/}" -k "${KV}" -F filename $(mod_dep_list) 2>/dev/null || echo
+	) | sort | uniq )
+
+	if [ ${#modlist[@]} -gt 0 ]
 	then
-		print_warning 1 "$(get_indent 2)modules: ${n_copied_modules} modules copied. Is that correct?"
+		print_info 2 "$(get_indent 2)modules: >> Copying modules from '${modules_srcdir}' to initramfs ..."
+		cp -ax --parents --target-directory "${modules_dstdir}" "${modlist[@]}" 2>/dev/null \
+			|| gen_die "Failed to copy modules!"
+		print_info 2 "$(get_indent 2)modules: ${#modlist[@]} modules copied!"
 	else
-		print_info 2 "$(get_indent 2)modules: ${n_copied_modules} modules copied!"
+		print_warning 1 "$(get_indent 2)modules: 0 modules copied. Is that correct?"
 	fi
 
 	cp -ax --parents --target-directory "${modules_dstdir}" modules* 2>/dev/null \
@@ -2108,16 +2143,6 @@ append_data() {
 create_initramfs() {
 	print_info 1 "initramfs: >> Initializing ..."
 
-	if ! isTrue "${BUILD_KERNEL}"
-	then
-		# Check early for suitable kmod
-		determine_KEXT
-		if ! isTrue "$(is_kext_supported_by_kmod "${KEXT}")"
-		then
-			gen_die "${KMOD_CMD} does not support chosen module compression algorithm. Please re-emerge sys-apps/kmod with USE=$(get_kext_kmod_use_flag "${KEXT}") enabled or adjust CONFIG_MODULE_COMPRESS_* kernel option!"
-		fi
-	fi
-
 	# Create empty cpio
 	CPIO_ARCHIVE="${TMPDIR}/${GK_FILENAME_TEMP_INITRAMFS}"
 	append_data 'devices' # WARNING, must be first!
diff --git a/gen_moddeps.sh b/gen_moddeps.sh
index 89a562b7..f175b720 100755
--- a/gen_moddeps.sh
+++ b/gen_moddeps.sh
@@ -1,69 +1,63 @@
 #!/bin/bash
 # $Id$
 
+mod_dep_list() {
+	if [ ! -f "${TEMP}/moddeps" ]
+	then
+		gen_dep_list > "${TEMP}/moddeps"
+	fi
+
+	cat "${TEMP}/moddeps"
+}
+
 gen_dep_list() {
+	local moddir="${KERNEL_MODULES_PREFIX%/}/lib/modules/${KV}"
+
 	if isTrue "${ALLRAMDISKMODULES}"
 	then
-		strip_mod_paths $(find "${KERNEL_MODULES_PREFIX%/}/lib/modules/${KV}" -name "*${KEXT}") | sort
+		cat "${moddir}/modules.builtin"
+		cat "${moddir}/modules.order"
 	else
-		rm -f "${TEMP}/moddeps" >/dev/null
+		local -a modlist=()
 
-		local group_modules
-		for group_modules in ${!MODULES_*} GK_INITRAMFS_ADDITIONAL_KMODULES
+		local mygroups
+		for mygroups in ${!MODULES_*} GK_INITRAMFS_ADDITIONAL_KMODULES
 		do
-			gen_deps ${!group_modules}
+			modlist+=( ${!mygroups} )
 		done
 
-		# Only list each module once
-		if [ -f "${TEMP}/moddeps" ]
-		then
-			cat "${TEMP}/moddeps" | sort | uniq
-		fi
-	fi
-}
+		modlist=( $(printf '%s\n' "${modlist[@]}" | sort | uniq) )
 
-gen_deps() {
-	local modlist
-	local deps
+		modlist+=( $(
+			local -a rxargs=( "${modlist[@]}" )
 
-	local x
-	for x in ${*}
-	do
-		echo ${x} >> "${TEMP}/moddeps"
-		modlist=$(modules_dep_list ${x})
-		if [ "${modlist}" != "" -a "${modlist}" != " " ]
-		then
-			deps=$(strip_mod_paths ${modlist})
-		else
-			deps=""
-		fi
+			rxargs=( "${rxargs[@]/#/-ealias\ }" )
+			rxargs=( "${rxargs[@]/%/\ }" )
 
-		local y
-		for y in ${deps}
-		do
-			echo ${y} >> "${TEMP}/moddeps"
-		done
-	done
-}
-
-modules_dep_list() {
-	if [ -f "${KERNEL_MODULES_PREFIX%/}/lib/modules/${KV}/modules.dep" ]
-	then
-		grep -F -- "/${1}${KEXT}:" "${KERNEL_MODULES_PREFIX%/}/lib/modules/${KV}/modules.dep" | cut -d\:  -f2
-	fi
-}
+			cat "${moddir}/modules.alias" \
+				| grep -F "${rxargs[@]}" \
+				| cut -d' ' -f3-
+		) )
 
-# Pass module deps list
-strip_mod_paths() {
-	local x
-	local ret
-	local myret
+		modlist=( $(printf '%s\n' "${modlist[@]}" | sort | uniq) )
 
-	for x in ${*}
-	do
-		ret=$(basename ${x} | cut -d. -f1)
-		myret="${myret} ${ret}"
-	done
-
-	echo "${myret}"
+		local mydeps mymod
+		while IFS=" " read -r -u 3 mymod mydeps
+		do
+			echo ${mymod%:}
+			printf '%s\n' ${mydeps}
+		done 3< <(
+			local -a rxargs=( "${modlist[@]}" )
+
+			rxargs=( "${rxargs[@]/#/-e\/}" )
+			rxargs=( "${rxargs[@]/%/${KEXT}:}" )
+
+			cat "${moddir}/modules.builtin" \
+				| xargs printf '%s:\n' \
+				| grep -F "${rxargs[@]}"
+
+			cat "${moddir}/modules.dep" \
+				| grep -F "${rxargs[@]}"
+		)
+	fi | xargs basename -s "${KEXT}" | sort | uniq
 }
diff --git a/genkernel b/genkernel
index b32d0a33..83313eab 100755
--- a/genkernel
+++ b/genkernel
@@ -352,6 +352,16 @@ if isTrue "${BUILD_RAMDISK}"
 then
 	print_info 1 '' 1 0
 
+	if isTrue "${FIRMWARE}" || isTrue "${RAMDISKMODULES}"
+	then
+		# Check for suitable kmod
+		determine_KEXT
+		if ! isTrue "$(is_kext_supported_by_kmod "${KEXT}")"
+		then
+			gen_die "${KMOD_CMD} does not support chosen module compression algorithm. Please re-emerge sys-apps/kmod with USE=$(get_kext_kmod_use_flag "${KEXT}") enabled or adjust CONFIG_MODULE_COMPRESS_* kernel option!"
+		fi
+	fi
+
 	# Compile initramfs
 	create_initramfs
 else
diff --git a/genkernel.conf b/genkernel.conf
index dae0b421..c4b44ea5 100644
--- a/genkernel.conf
+++ b/genkernel.conf
@@ -144,15 +144,19 @@ NOCOLOR="false"
 # Will conflict with sys-kernel/linux-firmware package
 #FIRMWARE_INSTALL="no"
 
-# Add firmware(s) to initramfs
+# Include full contents of FIRMWARE_DIR
+# (if FIRMWARE option below is set to YES).
+#ALLFIRMWARE="no"
+
+# Add firmware(s) to initramfs required by copied modules
 #FIRMWARE="no"
 
 # Specify directory to pull from
 #FIRMWARE_DIR="/lib/firmware"
 
 # Specify a comma-separated list of firmware files or directories to include,
-# relative to FIRMWARE_DIR.  If empty or unset, the full contents of 
-# FIRMWARE_DIR will be included (if FIRMWARE option above is set to YES).
+# relative to FIRMWARE_DIR (if FIRMWARE option above is set to YES
+# and ALLFIRMWARE is set to NO).
 #FIRMWARE_FILES=""
 
 # Add new kernel to grub