From 140824e69c763f3694d9cad64904909cf2a85853 Mon Sep 17 00:00:00 2001 From: NikLeberg Date: Sat, 22 Jan 2022 20:48:28 +0000 Subject: [PATCH 1/4] buildsystem/pkg: expand paths early --- Makefile.base | 4 ++-- Makefile.dep | 2 +- Makefile.include | 25 ++++++++++++++----------- makefiles/dependency_resolution.inc.mk | 4 ++++ makefiles/vars.inc.mk | 1 + 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Makefile.base b/Makefile.base index 0132e8c7e7db..cfc060c97348 100644 --- a/Makefile.base +++ b/Makefile.base @@ -97,8 +97,8 @@ compile-commands: | $(DIRS:%=COMPILE-COMMANDS--%) $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH: $(TARGET_ARCH)) $(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH_LLVM: $(TARGET_ARCH_LLVM)) -# include makefile snippets for packages in $(USEPKG) that modify GENSRC: --include $(USEPKG:%=$(RIOTPKG)/%/Makefile.gensrc) +# include makefile snippets for packages in $(PKG_PATHS) that modify GENSRC: +-include $(PKG_PATHS:%=%Makefile.gensrc) GENOBJC := $(GENSRC:%.c=%.o) OBJC_LTO := $(SRC:%.c=$(BINDIR)/$(MODULE)/%.o) diff --git a/Makefile.dep b/Makefile.dep index 6e0f815619a3..4b3df3a74294 100644 --- a/Makefile.dep +++ b/Makefile.dep @@ -20,7 +20,7 @@ include $(RIOTBASE)/drivers/Makefile.dep -include $(sort $(USEMODULE:%=$(RIOTBASE)/drivers/%/Makefile.dep)) # pull dependencies from packages --include $(USEPKG:%=$(RIOTPKG)/%/Makefile.dep) +-include $(PKG_PATHS:%=%Makefile.dep) ifneq (,$(filter mpu_stack_guard,$(USEMODULE))) FEATURES_REQUIRED += cortexm_mpu diff --git a/Makefile.include b/Makefile.include index 97810cece760..e590eabef932 100644 --- a/Makefile.include +++ b/Makefile.include @@ -427,6 +427,10 @@ ifeq (1,$(TEST_KCONFIG)) USEMODULE := $(KCONFIG_MODULES) KCONFIG_PACKAGES := $(call lowercase,$(patsubst CONFIG_PACKAGE_%,%,$(filter CONFIG_PACKAGE_%,$(.VARIABLES)))) USEPKG := $(KCONFIG_PACKAGES) + + # Locate used packages in $(RIOTPKG). + PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\ + $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile))))) else # always select provided architecture features FEATURES_REQUIRED += $(filter arch_%,$(FEATURES_PROVIDED)) @@ -570,8 +574,8 @@ include $(RIOTBASE)/sys/Makefile.include # include Makefile.includes of each driver modules if they exist -include $(USEMODULE:%=$(RIOTBASE)/drivers/%/Makefile.include) -# include Makefile.includes for packages in $(USEPKG) --include $(USEPKG:%=$(RIOTPKG)/%/Makefile.include) +# include Makefile.includes for packages in $(PKG_PATHS) +-include $(PKG_PATHS:%=%Makefile.include) # include external modules configuration -include $(EXTERNAL_MODULE_PATHS:%=%/Makefile.include) @@ -771,31 +775,30 @@ endif # The `clean` needs to be serialized before everything else. all $(BASELIBS) $(ARCHIVES) $(BUILDDEPS) ..in-docker-container: | $(CLEAN) -.PHONY: pkg-prepare pkg-build pkg-build-% +.PHONY: pkg-prepare pkg-build pkg-prepare: - -@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i prepare ; done + -@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) prepare $(NEWLINE)) -pkg-build: $(USEPKG:%=pkg-build-%) -pkg-build-%: $(BUILDDEPS) - $(QQ)"$(MAKE)" -C $(RIOTPKG)/$* +pkg-build: $(BUILDDEPS) + $(foreach dir,$(PKG_PATHS),$(QQ)"$(MAKE)" -C $(dir) $(NEWLINE)) clean: ifndef MAKE_RESTARTS - -@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i clean ; done + -@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) clean $(NEWLINE)) -@rm -rf $(BINDIR) -@rm -rf $(SCANBUILD_OUTPUTDIR) endif # Remove intermediates, but keep the .elf, .hex and .map etc. clean-intermediates: - -@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done + -@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE)) -@rm -rf $(BINDIR)/*.a $(BINDIR)/*/ clean-pkg: - -@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done + -@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE)) distclean: - -@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done + -@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE)) -@rm -rf $(BINDIRBASE) # Include PROGRAMMER_FLASH/PROGRAMMER_RESET variables diff --git a/makefiles/dependency_resolution.inc.mk b/makefiles/dependency_resolution.inc.mk index a3a76223c223..55a142076c32 100644 --- a/makefiles/dependency_resolution.inc.mk +++ b/makefiles/dependency_resolution.inc.mk @@ -6,6 +6,10 @@ EXTERNAL_MODULE_PATHS := $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\ $(foreach mod,$(USEMODULE),$(dir $(wildcard $(dir)/$(mod)/Makefile))))) +# Locate used packages in $(RIOTPKG). +PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\ + $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile))))) + # Back up current state to detect changes OLD_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED) diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk index c8607d295bd1..70df740c537a 100644 --- a/makefiles/vars.inc.mk +++ b/makefiles/vars.inc.mk @@ -45,6 +45,7 @@ export BINDIR # This is the folder where the application should b export CARGO_TARGET_DIR # This is the folder where Rust parts of the application should be built in. export BUILD_DIR # This is the base folder to store common build files and artifacts, e.g. test results. export APPDIR # The base folder containing the application +export PKG_PATHS # List of absolute paths where packages of $(USEPKG) can be found export PKGDIRBASE # The base folder for building packages export PYTHONPATH # Python default search path for module filesi, with RIOT specific packages From 044701d3cca2e19cd121bffcec18cf00aa67609a Mon Sep 17 00:00:00 2001 From: NikLeberg Date: Thu, 27 Jan 2022 10:56:17 +0000 Subject: [PATCH 2/4] pkg/esp8266_sdk: add requirements to PKG_PREPARE The shared `build-libs` directory needs to be available for modules/packages that depend on the SDK before that package is eventually compiled. Packages are downloaded, patched and prepared before any module is compiled. By adding the directory creation and header as a dependency of `PKG_PREPARE` we make sure the rule is run before compilation starts. --- pkg/esp8266_sdk/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/esp8266_sdk/Makefile b/pkg/esp8266_sdk/Makefile index 1453c6300622..59cde6131d93 100644 --- a/pkg/esp8266_sdk/Makefile +++ b/pkg/esp8266_sdk/Makefile @@ -20,7 +20,9 @@ ESP_SDK_LIBS = $(addprefix $(ESP8266_SDK_BUILD_DIR)/, $(ESP_SDK_COMPONENT_LIBS)) all: $(ESP_SDK_LIBS) $(ESP8266_SDK_BUILD_DIR)/esp8266_idf_version.h -$(ESP8266_SDK_BUILD_DIR): +$(PKG_PREPARED): $(ESP8266_SDK_BUILD_DIR) + +$(ESP8266_SDK_BUILD_DIR): $(PKG_PATCHED) $(Q)mkdir -p $(ESP8266_SDK_BUILD_DIR) # Set the SDK version from the SDK hash/tag. For example "v3.1-51-g913a06a9". From 18dca4851066fe984ae9bf73aa811896525c8fc8 Mon Sep 17 00:00:00 2001 From: NikLeberg Date: Fri, 28 Jan 2022 12:41:18 +0000 Subject: [PATCH 3/4] pkg/esp32_sdk: add requirements to PKG_PREPARE The shared `build-libs` directory needs to be available for modules/packages that depend on the SDK before that package is eventually compiled. This also includes header files such as `esp32_idf_version.h`. Packages are downloaded, patched, prepared before any module is compiled. By adding the directory creation and header as a dependency of `PKG_PREPARE` we make sure the rules are ran before compilation starts. --- pkg/esp32_sdk/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/esp32_sdk/Makefile b/pkg/esp32_sdk/Makefile index 0d0c813b5d99..1538c7d02c9b 100644 --- a/pkg/esp32_sdk/Makefile +++ b/pkg/esp32_sdk/Makefile @@ -20,11 +20,13 @@ ESP32_SDK_LIBS = $(addprefix $(ESP32_SDK_BUILD_DIR)/, $(ESP32_SDK_COMPONENT_LIBS all: $(ESP32_SDK_LIBS) $(ESP32_SDK_VER_FILE) -$(ESP32_SDK_BUILD_DIR): +$(PKG_PREPARED): $(ESP32_SDK_BUILD_DIR) $(ESP32_SDK_VER_FILE) + +$(ESP32_SDK_BUILD_DIR): $(PKG_PATCHED) $(Q)mkdir -p $(ESP32_SDK_BUILD_DIR) # Set the SDK version from the SDK hash/tag. For example "v3.1-51-g913a06a9". -$(ESP32_SDK_VER_FILE): +$(ESP32_SDK_VER_FILE): $(PKG_PATCHED) | $(ESP32_SDK_BUILD_DIR) $(Q)echo "#define IDF_VER \"$(ESP32_SDK_VER_CMD)\"" > $@ $(ESP32_SDK_BUILD_DIR)/lib%.a: \ From e5365adb93ac8acbeec0bb8dc8832916298edc30 Mon Sep 17 00:00:00 2001 From: NikLeberg Date: Fri, 28 Jan 2022 12:50:04 +0000 Subject: [PATCH 4/4] pkg/esp32_sdk_libs: add requirements to PKG_PREPARE The shared `build-libs` directory needs to be available for modules/packages that depend on the SDK before that package is eventually compiled. Packages are downloaded, patched, prepared before any module is compiled. By adding the directory creation as a dependency of `PKG_PREPARE` we make sure the rule is run before compilation starts. --- pkg/esp32_sdk_libs/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/esp32_sdk_libs/Makefile b/pkg/esp32_sdk_libs/Makefile index 56760a4e25e6..0ce50096dd4f 100644 --- a/pkg/esp32_sdk_libs/Makefile +++ b/pkg/esp32_sdk_libs/Makefile @@ -31,7 +31,9 @@ ESP32_SDK_LIBS = $(addprefix $(ESP32_SDK_BUILD_DIR)/, $(ESP32_SDK_COMPONENT_LIBS all: $(ESP32_SDK_LIBS) -$(ESP32_SDK_BUILD_DIR): +$(PKG_PREPARED): $(ESP32_SDK_BUILD_DIR) + +$(ESP32_SDK_BUILD_DIR): $(PKG_PATCHED) $(Q)mkdir -p $(ESP32_SDK_BUILD_DIR) $(ESP32_SDK_BUILD_DIR)/lib%.a: \