[XEN PATCH v8 33/47] build: adding out-of-tree support to the xen build

[Anthony PERARD <anthony.perard@xxxxxxxxxx>]

This implement out-of-tree support, there's two ways to create an
out-of-tree build tree (after that, `make` in that new directory
works):
    make O=build
    mkdir build; cd build; make -f ../Makefile
also works with an absolute path for both.

This implementation only works if the source tree is clean, as we use
VPATH.

This patch copies most new code with handling out-of-tree build from
Linux v5.12.

Signed-off-by: Anthony PERARD <anthony.perard@xxxxxxxxxx>
---

Notes:
    v8:
    - replace script/mkmakefile by cmd_makefile like it's been done in Linux
      5.13.
    - fix GNU Make version, replace few 0.81 by 3.81.
    - reorder include header path in CFLAGS ( -I ), to have the $(objtree)
      paths listed before the $(srctree) paths. This will be usefull later
      if we allow to build out-of-tree from a dirty source tree.
    - make use of -iquote
    - sometime, add -iquote path even when not necessary when building in-tree.
    - extract code movement into a separate patch
    - make use of the new "$(srcdir)" shortcut
    - split command line in headers*.chk target into more lines

 xen/Makefile                 | 141 +++++++++++++++++++++++++++++++----
 xen/Rules.mk                 |  11 ++-
 xen/arch/x86/arch.mk         |   5 +-
 xen/arch/x86/boot/Makefile   |   6 ++
 xen/common/efi/efi_common.mk |   1 +
 xen/include/Makefile         |  11 ++-
 xen/test/livepatch/Makefile  |   2 +
 xen/xsm/flask/Makefile       |   3 +-
 xen/xsm/flask/ss/Makefile    |   1 +
 9 files changed, 161 insertions(+), 20 deletions(-)

diff --git a/xen/Makefile b/xen/Makefile
index bfc96562c5ec..995815e82235 100644
--- a/xen/Makefile
+++ b/xen/Makefile
@@ -1,3 +1,7 @@
+# $(lastword,) for GNU Make older than 3.81
+lastword = $(word $(words $(1)),$(1))
+this-makefile := $(call lastword,$(MAKEFILE_LIST))
+
 # This is the correct place to edit the build version.
 # All other places this is stored (eg. compile.h) should be autogenerated.
 export XEN_VERSION       = 4
@@ -19,6 +23,13 @@ export PYTHON                ?= $(PYTHON_INTERPRETER)
 
 export CHECKPOLICY     ?= checkpolicy
 
+$(if $(filter __%, $(MAKECMDGOALS)), \
+    $(error targets prefixed with '__' are only for internal use))
+
+# That's our default target when none is given on the command line
+PHONY := __all
+__all:
+
 # Do not use make's built-in rules and variables
 MAKEFLAGS += -rR
 
@@ -31,9 +42,6 @@ export CC CXX LD
 
 export TARGET := xen
 
-.PHONY: default
-default: build
-
 .PHONY: dist
 dist: install
 
@@ -112,16 +120,88 @@ endif
 
 export quiet Q KBUILD_VERBOSE
 
+# $(realpath,) for GNU Make older than 3.81
+realpath = $(wildcard $(foreach file,$(1),$(shell cd -P $(dir $(file)) && echo 
"$$PWD/$(notdir $(file))")))
+
+ifeq ("$(origin O)", "command line")
+    KBUILD_OUTPUT := $(O)
+endif
+
+ifneq ($(KBUILD_OUTPUT),)
+# Make's built-in functions such as $(abspath ...), $(realpath ...) cannot
+# expand a shell special character '~'. We use a somewhat tedious way here.
+abs_objtree := $(shell mkdir -p $(KBUILD_OUTPUT) && cd $(KBUILD_OUTPUT) && pwd)
+$(if $(abs_objtree),, \
+     $(error failed to create output directory "$(KBUILD_OUTPUT)"))
+
+# $(realpath ...) resolves symlinks
+abs_objtree := $(call realpath,$(abs_objtree))
+else
 abs_objtree := $(CURDIR)
-abs_srctree := $(CURDIR)
+endif
+
+ifeq ($(abs_objtree),$(CURDIR))
+# Suppress "Entering directory ..." unless we are changing the work directory.
+MAKEFLAGS += --no-print-directory
+else
+need-sub-make := 1
+endif
+
+abs_srctree := $(call realpath,$(dir $(this-makefile)))
+
+ifneq ($(words $(subst :, ,$(abs_srctree))), 1)
+$(error source directory cannot contain spaces or colons)
+endif
+
+ifneq ($(abs_srctree),$(abs_objtree))
+# Look for make include files relative to root of kernel src
+#
+# This does not become effective immediately because MAKEFLAGS is re-parsed
+# once after the Makefile is read. We need to invoke sub-make.
+MAKEFLAGS += --include-dir=$(abs_srctree)
+need-sub-make := 1
+endif
 
 export abs_srctree abs_objtree
+export root-make-done := y
+
+ifeq ($(need-sub-make),1)
+
+PHONY += $(MAKECMDGOALS) __sub-make
+
+$(filter-out $(this-makefile), $(MAKECMDGOALS)) __all: __sub-make
+       @:
+
+# Invoke a second make in the output directory, passing relevant variables
+__sub-make:
+       $(Q)$(MAKE) -C $(abs_objtree) -f $(abs_srctree)/Makefile $(MAKECMDGOALS)
+
+endif # need-sub-make
+endif # root-make-done
+
+# We process the rest of the Makefile if this is the final invocation of make
+ifeq ($(need-sub-make),)
+
+ifeq ($(abs_srctree),$(abs_objtree))
+    # building in the source tree
+    srctree := .
+    building_out_of_srctree :=
+else
+    ifeq ($(abs_srctree)/,$(dir $(abs_objtree)))
+        # building in a subdirectory of the source tree
+        srctree := ..
+    else
+        srctree := $(abs_srctree)
+    endif
+    building_out_of_srctree := 1
+endif
 
-srctree := .
 objtree := .
-export srctree objtree
+VPATH := $(srctree)
+
+export building_out_of_srctree srctree objtree VPATH
 
-export XEN_ROOT := $(CURDIR)/..
+export XEN_ROOT := $(abs_srctree)/..
 
 # To make sure we do not include .config for any of the *config targets
 # catch them early, and hand them over to tools/kconfig/Makefile
@@ -204,9 +284,6 @@ endif
 
 export XEN_HAS_CHECKPOLICY := $(call success,$(CHECKPOLICY) -h 2>&1 | grep -q 
xen)
 
-export root-make-done := y
-endif # root-make-done
-
 # ===========================================================================
 # Rules shared between *config targets and build targets
 
@@ -214,6 +291,37 @@ PHONY += tools_fixdep
 tools_fixdep:
        $(Q)$(MAKE) $(build)=tools tools/fixdep
 
+PHONY += outputmakefile
+# Before starting out-of-tree build, make sure the source tree is clean.
+# outputmakefile generates a Makefile in the output directory, if using a
+# separate output directory. This allows convenient use of make in the
+# output directory.
+# At the same time when output Makefile generated, generate .gitignore to
+# ignore whole output directory
+
+quiet_cmd_makefile = GEN     Makefile
+cmd_makefile = { \
+    echo "\# Automatically generated by $(srctree)/Makefile: don't edit"; \
+    echo "include $(srctree)/Makefile"; \
+    } > Makefile
+
+outputmakefile:
+ifdef building_out_of_srctree
+       $(Q)if [ -f $(srctree)/.config -o \
+                -d $(srctree)/include/config -o \
+                -d $(srctree)/include/generated ]; then \
+               echo >&2 "***"; \
+               echo >&2 "*** The source tree is not clean, please run 
'make$(if $(findstring command line, $(origin XEN_TARGET_ARCH)), 
XEN_TARGET_ARCH=$(XEN_TARGET_ARCH)) distclean'"; \
+               echo >&2 "*** in $(abs_srctree)";\
+               echo >&2 "***"; \
+               false; \
+       fi
+       $(Q)ln -fsn $(srctree) source
+       $(call cmd,makefile)
+       $(Q)test -e .gitignore || \
+       { echo "# this is build directory, ignore it"; echo "*"; } > .gitignore
+endif
+
 ifeq ($(config-build),y)
 # ===========================================================================
 # *config targets only - make sure prerequisites are updated, and descend
@@ -229,13 +337,13 @@ filechk_kconfig_allconfig = \
 .allconfig.tmp: FORCE
        set -e; { $(call filechk_kconfig_allconfig); } > $@
 
-config: tools_fixdep FORCE
+config: tools_fixdep outputmakefile FORCE
        $(Q)$(MAKE) $(build)=tools/kconfig $@
 
 # Config.mk tries to include .config file, don't try to remake it
 %/.config: ;
 
-%config: .allconfig.tmp tools_fixdep FORCE
+%config: .allconfig.tmp tools_fixdep outputmakefile FORCE
        $(Q)$(MAKE) $(build)=tools/kconfig KCONFIG_ALLCONFIG=$< $@
 
 else # !config-build
@@ -315,6 +423,10 @@ CFLAGS += -flto
 LDFLAGS-$(CONFIG_CC_IS_CLANG) += -plugin LLVMgold.so
 endif
 
+ifdef building_out_of_srctree
+    CFLAGS += -I$(objtree)/include
+    CFLAGS += -I$(objtree)/arch/$(TARGET_ARCH)/include
+endif
 CFLAGS += -I$(srctree)/include
 CFLAGS += -I$(srctree)/arch/$(TARGET_ARCH)/include
 
@@ -341,6 +453,8 @@ export CFLAGS_UBSAN
 
 endif # need-config
 
+__all: build
+
 main-targets := build install uninstall clean distclean MAP
 .PHONY: $(main-targets)
 ifneq ($(XEN_TARGET_ARCH),x86_32)
@@ -443,7 +557,7 @@ $(TARGET).gz: $(TARGET)
        gzip -n -f -9 < $< > $@.new
        mv $@.new $@
 
-$(TARGET): FORCE
+$(TARGET): outputmakefile FORCE
        $(Q)$(MAKE) $(build)=tools
        $(Q)$(MAKE) $(build)=. include/xen/compile.h
        $(Q)$(MAKE) $(build)=include all
@@ -513,6 +627,7 @@ cloc:
        done | cloc --list-file=-
 
 endif #config-build
+endif # need-sub-make
 
 PHONY += FORCE
 FORCE:
diff --git a/xen/Rules.mk b/xen/Rules.mk
index 20dedce06cd8..73d5b82ab2e7 100644
--- a/xen/Rules.mk
+++ b/xen/Rules.mk
@@ -37,7 +37,7 @@ SPECIAL_DATA_SECTIONS := rodata $(foreach a,1 2 4 8 16, \
                          $(foreach r,rel rel.ro,data.$(r).local)
 
 # The filename build.mk has precedence over Makefile
-include $(firstword $(wildcard $(src)/build.mk) $(src)/Makefile)
+include $(firstword $(wildcard $(srcdir)/build.mk) $(srcdir)/Makefile)
 
 # Linking
 # ---------------------------------------------------------------------------
@@ -328,6 +328,15 @@ existing-targets := $(wildcard $(sort $(targets)))
 
 -include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd)
 
+# Create directories for object files if they do not exist
+obj-dirs := $(sort $(patsubst %/,%, $(dir $(targets))))
+# If targets exist, their directories apparently exist. Skip mkdir.
+existing-dirs := $(sort $(patsubst %/,%, $(dir $(existing-targets))))
+obj-dirs := $(strip $(filter-out $(existing-dirs), $(obj-dirs)))
+ifneq ($(obj-dirs),)
+$(shell mkdir -p $(obj-dirs))
+endif
+
 # Declare the contents of the PHONY variable as phony.  We keep that
 # information in a variable so we can use it in if_changed and friends.
 .PHONY: $(PHONY)
diff --git a/xen/arch/x86/arch.mk b/xen/arch/x86/arch.mk
index 7cfc9fd3bb1c..554ac341522c 100644
--- a/xen/arch/x86/arch.mk
+++ b/xen/arch/x86/arch.mk
@@ -62,6 +62,9 @@ ifneq ($(CONFIG_PV_SHIM_EXCLUSIVE),y)
 
 efi-check-o := arch/x86/efi/check.o
 
+# Create the directory for out-of-tree build
+$(shell mkdir -p $(dir $(efi-check-o)))
+
 # Check if the compiler supports the MS ABI.
 XEN_BUILD_EFI := $(call if-success,$(CC) $(CFLAGS) -c 
$(srctree)/$(efi-check-o:.o=.c) -o $(efi-check-o),y)
 
@@ -101,6 +104,6 @@ export EFI_LDFLAGS
 endif
 
 # Set up the assembler include path properly for older toolchains.
-CFLAGS += -Wa,-I$(srctree)/include
+CFLAGS += -Wa,-I$(objtree)/include -Wa,-I$(srctree)/include
 
 ALL_OBJS-y := arch/x86/boot/built_in.o arch/x86/efi/built_in.o $(ALL_OBJS-y)
diff --git a/xen/arch/x86/boot/Makefile b/xen/arch/x86/boot/Makefile
index 5395aae5d57e..04ee9da83333 100644
--- a/xen/arch/x86/boot/Makefile
+++ b/xen/arch/x86/boot/Makefile
@@ -7,6 +7,9 @@ targets += $(head-objs:.S=.o)
 
 head-objs := $(addprefix $(obj)/, $(head-objs))
 
+ifdef building_out_of_srctree
+$(obj)/head.o: CFLAGS-y += -iquote $(obj)
+endif
 $(obj)/head.o: $(head-objs)
 
 $(head-objs:.S=.lnk): LDFLAGS_DIRECT := $(subst x86_64,i386,$(LDFLAGS_DIRECT))
@@ -14,6 +17,9 @@ $(head-objs:.S=.lnk): LDFLAGS_DIRECT := $(subst 
x86_64,i386,$(LDFLAGS_DIRECT))
 CFLAGS_x86_32 := $(subst -m64,-m32 -march=i686,$(XEN_COMMON_CFLAGS))
 $(call cc-options-add,CFLAGS_x86_32,CC,$(EMBEDDED_EXTRA_CFLAGS))
 CFLAGS_x86_32 += -Werror -fno-builtin -g0 -msoft-float
+ifdef building_out_of_srctree
+CFLAGS_x86_32 += -I$(objtree)/include
+endif
 CFLAGS_x86_32 += -I$(srctree)/include
 
 # override for 32bit binaries
diff --git a/xen/common/efi/efi_common.mk b/xen/common/efi/efi_common.mk
index d2845fd6b3c8..d059bb30afa5 100644
--- a/xen/common/efi/efi_common.mk
+++ b/xen/common/efi/efi_common.mk
@@ -3,6 +3,7 @@ EFIOBJ-$(CONFIG_COMPAT) += compat.o
 
 CFLAGS-y += -fshort-wchar
 CFLAGS-y += -iquote $(srctree)/common/efi
+CFLAGS-y += -iquote $(srcdir)
 
 $(obj)/%.c: $(abs_srctree)/common/efi/%.c FORCE
        $(Q)ln -nfs $< $@
diff --git a/xen/include/Makefile b/xen/include/Makefile
index f64bc293c7f4..a3977a8bfccd 100644
--- a/xen/include/Makefile
+++ b/xen/include/Makefile
@@ -113,7 +113,8 @@ $(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) 
$(srcdir)/Makefile
        $(foreach i, $(filter %.h,$^),                                        \
            echo "#include "\"$(i)\"                                          \
            | $(CC) -x c -std=c99 -Wall -Werror                               \
-             -include stdint.h $(foreach j, $($(i)-prereq), -include $(j).h) \
+             -include stdint.h                                               \
+             $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include 
$(j).h) \
              -S -o /dev/null -                                               \
            || exit $$?; echo $(i) >> $@.new;)
        mv $@.new $@
@@ -127,8 +128,9 @@ $(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile
        $(foreach i, $(filter %.h,$^),                                        \
            echo "#include "\"$(i)\"                                          \
            | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__        \
-             -include stdint.h -include $(src)/public/xen.h                  \
-             $(foreach j, $($(i)-prereq), -include c$(j)) -S -o /dev/null -  \
+             -include stdint.h -include $(srcdir)/public/xen.h               \
+             $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include 
c$(j)) \
+             -S -o /dev/null -                                               \
            || exit $$?; echo $(i) >> $@.new;)
        mv $@.new $@
 
@@ -137,7 +139,8 @@ endif
 ifeq ($(XEN_TARGET_ARCH),x86_64)
 .PHONY: lib-x86-all
 lib-x86-all:
-       $(MAKE) -C $(obj)/xen/lib/x86 all
+       @mkdir -p $(obj)/xen/lib/x86
+       $(MAKE) -C $(obj)/xen/lib/x86 -f 
$(abs_srctree)/$(src)/xen/lib/x86/Makefile all
 
 all: lib-x86-all
 endif
diff --git a/xen/test/livepatch/Makefile b/xen/test/livepatch/Makefile
index ddb07371315e..c258ab0b5940 100644
--- a/xen/test/livepatch/Makefile
+++ b/xen/test/livepatch/Makefile
@@ -11,6 +11,8 @@ endif
 CODE_ADDR=$(shell nm --defined $(1) | grep $(2) | awk '{print "0x"$$1}')
 CODE_SZ=$(shell nm --defined -S $(1) | grep $(2) | awk '{ print "0x"$$2}')
 
+CFLAGS-y += -iquote $(obj)
+
 extra-y += xen_hello_world.livepatch
 xen_hello_world-objs := xen_hello_world_func.o xen_hello_world.o note.o 
xen_note.o modinfo.o
 $(obj)/xen_hello_world.o: $(obj)/config.h
diff --git a/xen/xsm/flask/Makefile b/xen/xsm/flask/Makefile
index a99038cb5722..d25312f4fa1c 100644
--- a/xen/xsm/flask/Makefile
+++ b/xen/xsm/flask/Makefile
@@ -4,7 +4,8 @@ obj-y += flask_op.o
 
 obj-y += ss/
 
-CFLAGS-y += -I$(obj)/include
+CFLAGS-y += -iquote $(obj)/include
+CFLAGS-y += -I$(srcdir)/include
 
 AWK = awk
 
diff --git a/xen/xsm/flask/ss/Makefile b/xen/xsm/flask/ss/Makefile
index aba1339f3808..ffe92ec19ed6 100644
--- a/xen/xsm/flask/ss/Makefile
+++ b/xen/xsm/flask/ss/Makefile
@@ -8,4 +8,5 @@ obj-y += services.o
 obj-y += conditional.o
 obj-y += mls.o
 
+CFLAGS-y += -iquote $(objtree)/xsm/flask/include
 CFLAGS-y += -I$(srctree)/xsm/flask/include
-- 
Anthony PERARD