new(pkg/driver): initial work towards kernel headers download leverag…

…ing driverkit library when building drivers.

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

go.mod

@@ -133,6 +133,7 @@ require (
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
@@ -190,6 +191,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
@@ -221,6 +223,7 @@ require (
 	github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect
 	github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 // indirect
 	github.com/redis/go-redis/v9 v9.3.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -295,6 +298,16 @@ require (
 	k8s.io/api v0.29.2 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	lukechampine.com/uint128 v1.3.0 // indirect
+	modernc.org/cc/v3 v3.41.0 // indirect
+	modernc.org/ccgo/v3 v3.16.15 // indirect
+	modernc.org/libc v1.24.1 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.1 // indirect
+	modernc.org/opt v0.1.3 // indirect
+	modernc.org/sqlite v1.25.0 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/release-utils v0.7.7 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -268,6 +268,8 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -507,6 +509,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -548,6 +552,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
@@ -658,6 +664,8 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnA
 github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
 github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
 github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -1144,6 +1152,34 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
+lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q=
+modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y=
+modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0=
+modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI=
+modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
+modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
+modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
+modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
+modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M=
+modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA=
+modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
+modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
+modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE=
 oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec=
 oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/driver/distro/cos.go

@@ -75,7 +75,7 @@ func (c *cos) customizeBuild(ctx context.Context,
 		return nil, err
 	}
 
-	currKernelDir := env[kernelDirEnv]
+	currKernelDir := env[drivertype.KernelDirEnv]
 
 	cosKernelDir := filepath.Join(currKernelDir, "usr", "src")
 	entries, err := os.ReadDir(cosKernelDir)
@@ -87,7 +87,7 @@ func (c *cos) customizeBuild(ctx context.Context,
 	}
 	cosKernelDir = filepath.Join(cosKernelDir, entries[0].Name())
 	// Override env key
-	env[kernelDirEnv] = cosKernelDir
+	env[drivertype.KernelDirEnv] = cosKernelDir
 
 	clangCompilerHeader := fmt.Sprintf("%s/include/linux/compiler-clang.h", cosKernelDir)
 	err = utils.ReplaceLineInFile(clangCompilerHeader, "#define randomized_struct_fields_start", "", 1)

pkg/driver/distro/distro.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 
 	"github.com/docker/docker/pkg/homedir"
+	"github.com/falcosecurity/driverkit/pkg/driverbuilder/builder"
 	"github.com/falcosecurity/driverkit/pkg/kernelrelease"
 	"golang.org/x/net/context"
 	"gopkg.in/ini.v1"
@@ -39,8 +40,6 @@ import (
 )
 
 const (
-	// DefaultFalcoRepo is the default repository provided by falcosecurity to download driver artifacts from.
-	kernelDirEnv            = "KERNELDIR"
 	kernelSrcDownloadFolder = "kernel-sources"
 )
 
@@ -127,6 +126,46 @@ func getOSReleaseDistro(kr *kernelrelease.KernelRelease) (Distro, error) {
 	return distro, nil
 }
 
+//nolint:gocritic // the method shall not be able to modify kr
+func loadKernelHeadersFromDk(distro string, kr kernelrelease.KernelRelease) ([]string, error) {
+	// Try to load kernel headers from driverkit. Don't error out if unable to.
+	b, err := builder.Factory(builder.Type(distro))
+	if err != nil {
+		return nil, nil
+	}
+
+	// Load minimum urls for the builder
+	minimumURLs := 1
+	if bb, ok := b.(builder.MinimumURLsBuilder); ok {
+		minimumURLs = bb.MinimumURLs()
+	}
+
+	// Fetch URLs
+	kernelheaders, err := b.URLs(kr)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check actually resolving URLs
+	kernelheaders, err = builder.GetResolvingURLs(kernelheaders)
+	if err != nil {
+		return nil, err
+	}
+	if len(kernelheaders) < minimumURLs {
+		return nil, fmt.Errorf("not enough headers packages found; expected %d, found %d", minimumURLs, len(kernelheaders))
+	}
+	return kernelheaders, nil
+}
+
+func downloadKernelHeaders(headers []string) (string, error) {
+	// TODO download all and mv everything under /tmp/kernel/usr/{lib,src}
+	// change driverkit?
+	for _, h := range headers {
+		fmt.Println(h)
+	}
+	return "", nil
+}
+
 func toURL(repo, driverVer, fileName, arch string) string {
 	return fmt.Sprintf("%s/%s/%s/%s", repo, url.QueryEscape(driverVer), arch, fileName)
 }
@@ -178,6 +217,20 @@ func Build(ctx context.Context,
 	if err != nil {
 		return "", err
 	}
+
+	// If customizeBuild did not set any KERNELDIR env variable,
+	// try to load kernel headers urls from driverkit.
+	if _, found := env[drivertype.KernelDirEnv]; !found {
+		headers, _ := loadKernelHeadersFromDk(d.String(), kr)
+		if headers != nil {
+			// We found the headers! Download them to `/tmp/kernel`.
+			path, err := downloadKernelHeaders(headers)
+			if err != nil {
+				env[drivertype.KernelDirEnv] = path
+			}
+		}
+	}
+
 	path, err := driverType.Build(ctx, printer, kr, driverName, driverVer, env)
 	if err != nil {
 		return "", err
@@ -215,10 +268,10 @@ func Download(ctx context.Context,
 	// Try to download from any specified repository,
 	// stopping at first successful http GET.
 	for _, repo := range repos {
-		url := toURL(repo, driverVer, driverFileName, kr.Architecture.ToNonDeb())
-		printer.Logger.Info("Trying to download a driver.", printer.Logger.Args("url", url))
+		driverURL := toURL(repo, driverVer, driverFileName, kr.Architecture.ToNonDeb())
+		printer.Logger.Info("Trying to download a driver.", printer.Logger.Args("url", driverURL))
 
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, driverURL, nil)
 		if err != nil {
 			printer.Logger.Warn("Error creating http request.", printer.Logger.Args("err", err))
 			continue
@@ -367,6 +420,6 @@ func downloadKernelSrc(ctx context.Context,
 	if err != nil {
 		return nil, err
 	}
-	env[kernelDirEnv] = fullKernelDir
+	env[drivertype.KernelDirEnv] = fullKernelDir
 	return env, nil
 }

pkg/driver/type/kmod.go

@@ -165,7 +165,8 @@ func (k *kmod) HasArtifacts() bool {
 	return true
 }
 
-func createDKMSMakeFile(gcc string) error {
+//nolint:goconst // ` $@` constification would be useless.
+func createDKMSMakeFile(gcc string, env map[string]string) error {
 	file, err := os.OpenFile("/tmp/falco-dkms-make", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o777) //nolint:gosec // we need the file to be executable
 	if err != nil {
 		return err
@@ -174,7 +175,11 @@ func createDKMSMakeFile(gcc string) error {
 
 	_, err = fmt.Fprintln(file, "#!/usr/bin/env bash")
 	if err == nil {
-		_, err = fmt.Fprintln(file, `make CC=`+gcc+` $@`)
+		if kernelDir, found := env[KernelDirEnv]; found {
+			_, err = fmt.Fprintln(file, `make CC=`+gcc+` KERNELDIR=`+kernelDir+` $@`)
+		} else {
+			_, err = fmt.Fprintln(file, `make CC=`+gcc+` $@`)
+		}
 	}
 	return err
 }
@@ -184,7 +189,7 @@ func (k *kmod) Build(ctx context.Context,
 	printer *output.Printer,
 	kr kernelrelease.KernelRelease,
 	driverName, driverVersion string,
-	_ map[string]string,
+	env map[string]string,
 ) (string, error) {
 	// Skip dkms on UEK hosts because it will always fail
 	if strings.Contains(kr.String(), "uek") {
@@ -213,7 +218,7 @@ func (k *kmod) Build(ctx context.Context,
 		}
 
 		printer.Logger.Info("Trying to dkms install module.", printer.Logger.Args("gcc", gcc))
-		err = createDKMSMakeFile(gcc)
+		err = createDKMSMakeFile(gcc, env)
 		if err != nil {
 			printer.Logger.Warn("Could not fill /tmp/falco-dkms-make content.")
 			continue

pkg/driver/type/type.go

@@ -26,7 +26,10 @@ import (
 )
 
 // TypeAuto enables a smart automatic driver selection logic instead of using a fixed driver type.
-const TypeAuto = "auto"
+const (
+	TypeAuto     = "auto"
+	KernelDirEnv = "KERNELDIR"
+)
 
 var driverTypes = map[string]DriverType{}