build system: Restructure dependency resolution

Makefile.dep

@@ -1,7 +1,3 @@
-#
-OLD_USEMODULE := $(sort $(USEMODULE))
-OLD_USEPKG := $(sort $(USEPKG))
-
 # include board specific application dependencies
 -include $(APPDIR)/Makefile.board.dep
 
@@ -1036,18 +1032,6 @@ FEATURES_OPTIONAL += periph_pm
 # always select provided architecture features
 FEATURES_REQUIRED += $(filter arch_%,$(FEATURES_PROVIDED))
 
-# all periph features correspond to a periph submodule
-# FEATURES_USED is defined in Makefile.features
-USEMODULE += $(filter periph_%,$(FEATURES_USED))
-
-# select cpu_check_address pseudomodule if the corresponding feature is used
-USEMODULE += $(filter cpu_check_address, $(FEATURES_USED))
-
-# include periph_common if any periph_* driver is used
-ifneq (,$(filter periph_%, $(USEMODULE)))
-  USEMODULE += periph_common
-endif
-
 ifneq (,$(filter ecc_%,$(USEMODULE)))
   USEMODULE += ecc
 endif
@@ -1057,44 +1041,3 @@ ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE)))
     DEFAULT_MODULE += test_utils_interactive_sync_shell
   endif
 endif
-
-# recursively catch transitive dependencies
-USEMODULE := $(sort $(USEMODULE))
-USEPKG := $(sort $(USEPKG))
-ifneq ($(OLD_USEMODULE) $(OLD_USEPKG),$(USEMODULE) $(USEPKG))
-  include $(RIOTBASE)/Makefile.dep
-else
-
-  # set all USED periph_% init modules as DEFAULT_MODULE
-  ifneq (,$(filter periph_init, $(USEMODULE)))
-      # To enable adding periph_% modules through the environment we cant use
-      # FEATURES_USED since the MODULE might be added directly as USEMODULE
-      PERIPH_MODULES_NO_INIT = periph_init% periph_common
-      PERIPH_MODULES = $(filter periph_%,$(USEMODULE))
-      PERIPH_INIT_MODULES = $(subst periph_,periph_init_,\
-        $(filter-out $(PERIPH_MODULES_NO_INIT),$(PERIPH_MODULES)))
-      DEFAULT_MODULE += $(PERIPH_INIT_MODULES)
-  endif
-
-  # add periph_init_% modules to USEMODULE unless disabled
-  ifneq (,$(filter periph_init, $(USEMODULE)))
-    USEMODULE += $(filter $(PERIPH_INIT_MODULES),\
-      $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
-  endif
-
-  # Add auto_init_% DEFAULT_MODULES. This is done after the recursive cach since
-  # none of these modules can trigger dependency resolution.
-  ifneq (,$(filter auto_init,$(USEMODULE)))
-    USEMODULE += $(filter auto_init_%,$(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
-  endif
-
-  # Add test_utils_interactive_sync_shell
-  ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE)))
-    USEMODULE += $(filter test_utils_interactive_sync_%, \
-      $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
-  endif
-
-  # Sort and remove duplicates
-  DEFAULT_MODULE := $(sort $(DEFAULT_MODULE))
-  USEMODULE := $(sort $(USEMODULE))
-endif

Makefile.features

@@ -1,4 +1,4 @@
-# Process FEATURES variables
+# Process provided FEATURES
 #
 # The board/board common are responsible for defining the CPU and CPU_MODEL
 # variables in their Makefile.features.
@@ -17,38 +17,3 @@ ifeq (,$(CPU))
 endif
 
 include $(RIOTCPU)/$(CPU)/Makefile.features
-
-
-# Resolve FEATURES_ variables
-# Their value will only be complete after resolving dependencies
-
-# Features that are required by the application but not provided by the BSP and
-# features that are used but blacklisted (prepended with "!").
-# Having features missing may case the build to fail.
-FEATURES_MISSING = $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_REQUIRED)))
-
-# Features that are only optional and not required at the same time.
-# The policy is to by default use by features if they are provided by the BSP.
-FEATURES_OPTIONAL_ONLY = $(sort $(filter-out $(FEATURES_REQUIRED),$(FEATURES_OPTIONAL)))
-FEATURES_OPTIONAL_USED = $(sort $(filter $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
-# Optional features that will not be used because they are not provided
-FEATURES_OPTIONAL_MISSING = $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
-
-# Features that are used for an application
-FEATURES_USED = $(sort $(FEATURES_REQUIRED) $(FEATURES_OPTIONAL_USED))
-
-# Used features that conflict when used together
-FEATURES_CONFLICTING = $(sort $(foreach conflict,$(FEATURES_CONFLICT),$(call _features_conflicting,$(conflict))))
-
-# Return conflicting features from the conflict string feature1:feature2
-#   $1: feature1:feature2
-#   Return the list of conflicting features
-_features_conflicting = $(if $(call _features_used_conflicting,$(subst :, ,$1)),$(subst :, ,$1))
-# Check if all features from the list are used
-#   $1: list of features that conflict together
-#   Return non empty on error
-_features_used_conflicting = $(filter $(words $1),$(words $(filter $(FEATURES_USED),$1)))
-
-# Features that are used by the application but blacklisted by the BSP.
-# Having blacklisted features may cause the build to fail.
-FEATURES_USED_BLACKLISTED = $(sort $(filter $(FEATURES_USED), $(FEATURES_BLACKLIST)))

Makefile.include

@@ -357,7 +357,7 @@ export CCACHE_CPP2=yes
 USEMODULE += $(filter-out $(DISABLE_MODULE), $(DEFAULT_MODULE))
 
 # process dependencies
-include $(RIOTBASE)/Makefile.dep
+include $(RIOTMAKE)/dependency_resolution.inc.mk
 
 ifeq ($(strip $(MCU)),)
 	MCU = $(CPU)

bootloaders/riotboot/Makefile

@@ -15,7 +15,7 @@ CFLAGS += -DRIOTBOOT
 # Disable unused modules
 CFLAGS += -DNDEBUG -DLOG_LEVEL=LOG_NONE
 DISABLE_MODULE += core_init core_msg core_panic
-DISABLE_MODULE += auto_init
+DISABLE_MODULE += auto_init auto_init_%
 
 # avoid using stdio
 USEMODULE += stdio_null

doc/doxygen/src/build-system-basics.md

@@ -55,6 +55,12 @@ For a `FEATURE` to be provided by a `board` it must meet 2 criteria, and for
 - `FEATURES_OPTIONAL` are "nice to have" `FEATURES`, not needed but useful. If
   available they are always included.
 
+- `FEATURES_REQUIRED_ANY` are `FEATURES` of which (at least) one of
+  is needed by a `MODULE` or `APPLICATION`. Alternatives are separated by
+  a pipe (`|`) in order of preference, e.g.:
+  `FEATURES_REQUIRED_ANY += arch_avr8|arch_native` if both are provide then
+  `arch_avr8` will be used.
+
 - `FEATURES_BLACKLIST` are `FEATURES` that can't be used by a `MODULE` or `APPLCIATION`.
   They are usually used for _hw_ characteristics like `arch_` to easily resolve
   unsupported configurations for a group.
@@ -66,9 +72,9 @@ For a `FEATURE` to be provided by a `board` it must meet 2 criteria, and for
 - `FEATURES_PROVIDED`, `FEATURES_CONFLICT` and `FEATURES_CONFLICT_MSG ` are
    defined in `Makefile.features`
 
-- `FEATURES_REQUIRED`, `FEATURES_OPTIONAL`, `FEATURES_BLACKLIST` are defined by
-   the application `Makefile` (`examples/%/Makefile`, `tests/%/Makfile`, etc.)
-   or in `Makefile.dep`
+- `FEATURES_REQUIRED`, `FEATURES_OPTIONAL`, `FEATURES_REQUIRED_ANY`,
+   and `FEATURES_BLACKLIST` are defined by the application `Makefile`
+   (`examples/%/Makefile`, `tests/%/Makefile`, etc.) or in `Makefile.dep`
 
 ## CPU/CPU_MODEL                                                            {#cpu}
 

drivers/Makefile.dep

@@ -779,8 +779,8 @@ ifneq (,$(filter ws281x_%,$(USEMODULE)))
 endif
 
 ifneq (,$(filter ws281x,$(USEMODULE)))
-  FEATURES_OPTIONAL += arch_avr8
-  FEATURES_OPTIONAL += arch_native
+  FEATURES_REQUIRED_ANY += arch_avr8|arch_native
+
   ifeq (,$(filter ws281x_%,$(USEMODULE)))
     ifneq (,$(filter arch_avr8,$(FEATURES_USED)))
       USEMODULE += ws281x_atmega

makefiles/dependencies_debug.inc.mk

@@ -39,12 +39,17 @@ DEPENDENCY_DEBUG_OUTPUT_DIR ?= $(CURDIR)
 
 # Save variables that are used for parsing dependencies
 _DEPS_DEBUG_VARS += BOARD CPU CPU_MODEL CPU_FAM
-_DEPS_DEBUG_VARS += FEATURES_PROVIDED _FEATURES_PROVIDED_SORTED FEATURES_REQUIRED _FEATURES_REQUIRED_SORTED FEATURES_OPTIONAL FEATURES_USED FEATURES_MISSING FEATURES_CONFLICT FEATURES_CONFLICTING
+_DEPS_DEBUG_VARS += FEATURES_PROVIDED _FEATURES_PROVIDED_SORTED
+_DEPS_DEBUG_VARS += FEATURES_REQUIRED _FEATURES_REQUIRED_SORTED
+_DEPS_DEBUG_VARS += FEATURES_REQUIRED_ANY _FEATURES_REQUIRED_ANY_SORTED
+_DEPS_DEBUG_VARS += FEATURES_OPTIONAL FEATURES_USED FEATURES_MISSING
+_DEPS_DEBUG_VARS += FEATURES_CONFLICT FEATURES_CONFLICTING
 _DEPS_DEBUG_VARS += USEMODULE DEFAULT_MODULE DISABLE_MODULE
 DEPS_DEBUG_VARS ?= $(_DEPS_DEBUG_VARS)
 
 _FEATURES_PROVIDED_SORTED = $(sort $(FEATURES_PROVIDED))
 _FEATURES_REQUIRED_SORTED = $(sort $(FEATURES_REQUIRED))
+_FEATURES_REQUIRED_ANY_SORTED = $(sort $(FEATURES_REQUIRED_ANY))
 
 file_save_dependencies_variables = $(call file_save_variable,$(DEPENDENCY_DEBUG_OUTPUT_DIR)/$1_$(BOARD),$(DEPS_DEBUG_VARS))
 # Remove file before to be sure appending is started with an empty file

makefiles/dependency_resolution.inc.mk

@@ -0,0 +1,41 @@
+# Perform a recursive dependency resolution: Include $(RIOTBASE)/Makefile.dep
+# until no new modules, pkgs, or features are pull in order to catch all
+# transient dependencies
+
+# Back up current state to detect changes
+OLD_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
+
+# pull in dependencies of the currently used modules and pkgs
+include $(RIOTBASE)/Makefile.dep
+
+# check if required features are provided and update $(FEATURES_USED)
+include $(RIOTMAKE)/features_check.inc.mk
+
+# translate used features into used module, where needed
+include $(RIOTMAKE)/features_modules.inc.mk
+
+# sort and de-duplicate used modules, pkgs, and features for comparison
+USEMODULE := $(sort $(USEMODULE))
+USEPKG := $(sort $(USEPKG))
+FEATURES_USED := $(sort $(FEATURES_USED))
+
+NEW_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
+
+# If set of used modules, pkgs, and features has changed during last run, run
+# again to recursively catch transitive dependencies
+ifneq ($(OLD_STATE),$(NEW_STATE))
+  include $(RIOTMAKE)/dependency_resolution.inc.mk
+else
+  # If module auto_init is not used, silently disable all of its submodules
+  ifeq (,$(filter auto_init,$(USEMODULE)))
+    DISABLE_MODULE += auto_init_%
+  endif
+
+  # add default modules again, as $(DEFAULT_MODULE) might have been extended
+  # during dependency processing
+  USEMODULE += $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE))
+
+  # Sort and de-duplicate used modules and default modules for readability
+  USEMODULE := $(sort $(USEMODULE))
+  DEFAULT_MODULE := $(sort $(DEFAULT_MODULE))
+endif

makefiles/features_check.inc.mk

@@ -0,0 +1,67 @@
+# Check if all required FEATURES are provided
+
+FEATURES_OPTIONAL_ONLY := $(sort $(filter-out $(FEATURES_REQUIRED),$(FEATURES_OPTIONAL)))
+FEATURES_OPTIONAL_USED := $(sort $(filter $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
+# Optional features that will not be used because they are not provided
+FEATURES_OPTIONAL_MISSING := $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
+
+# Features that are used without taking "one out of" dependencies into account
+FEATURES_USED_SO_FAR := $(sort $(FEATURES_REQUIRED) $(FEATURES_OPTIONAL_USED))
+
+# Features that are provided and not blacklisted
+FEATURES_USABLE := $(filter-out $(FEATURES_BLACKLIST),$(FEATURES_PROVIDED))
+
+# Additionally required features due to the "one out of" dependencies
+# Algorithm:
+#  - For each list in FEATURES_REQUIRED_ANY as "item":
+#     - Store the intersection of FEATURES_USED_SO_FAR and the features in
+#       "item" in "tmp"
+#     - Append the intersection of FEATURES_USABLE and the features in "item"
+#       to "tmp"
+#     - Append "item" to "tmp" (with pipes between features, e.g.
+#       "periph_spi|periph_i2c")
+#     - Append the first element of "tmp" to FEATURES_REQUIRED_ONE_OUT_OF
+#   ==> If one (or more) already used features is listed in item, this (or
+#       one of these) will be in the front of "tmp" and be taken
+#   ==> If one (or more) usable features is listed in item, this will come
+#       afterwards. If no no feature in item is used so for, one of the
+#       features supported and listed in item will be picked
+#   ==> At the end of the list item itself (with pipes between features) is the
+#       last item in "tmp". If no feature is item is supported or used, this
+#       will be the only item in "tmp" and be picked
+FEATURES_REQUIRED_ONE_OUT_OF := $(foreach item,\
+                                  $(FEATURES_REQUIRED_ANY),\
+                                  $(word 1,\
+                                    $(filter $(subst |, ,$(item)),\
+                                              $(FEATURES_USED_SO_FAR) \
+                                              $(FEATURES_USABLE)) \
+                                              $(item)))
+
+# Features that are required by the application but not provided by the BSP
+# Having features missing may case the build to fail.
+FEATURES_MISSING := $(sort \
+                      $(filter-out $(FEATURES_PROVIDED),\
+                        $(FEATURES_REQUIRED) $(FEATURES_REQUIRED_ONE_OUT_OF)))
+
+# Features that are used for an application
+FEATURES_USED := $(sort $(FEATURES_REQUIRED) \
+                        $(FEATURES_REQUIRED_ONE_OUT_OF) \
+                        $(FEATURES_OPTIONAL_USED))
+
+# Used features that conflict when used together
+FEATURES_CONFLICTING := $(sort $(foreach conflict,\
+                                 $(FEATURES_CONFLICT),\
+                                 $(call _features_conflicting,$(conflict))))
+
+# Return conflicting features from the conflict string feature1:feature2
+#   $1: feature1:feature2
+#   Return the list of conflicting features
+_features_conflicting = $(if $(call _features_used_conflicting,$(subst :, ,$1)),$(subst :, ,$1))
+# Check if all features from the list are used
+#   $1: list of features that conflict together
+#   Return non empty on error
+_features_used_conflicting = $(filter $(words $1),$(words $(filter $(FEATURES_USED),$1)))
+
+# Features that are used by the application but blacklisted by the BSP.
+# Having blacklisted features may cause the build to fail.
+FEATURES_USED_BLACKLISTED := $(sort $(filter $(FEATURES_USED), $(FEATURES_BLACKLIST)))

makefiles/features_modules.inc.mk

@@ -0,0 +1,27 @@
+# Add modules implementing used features
+#
+# This is done after the regular dependency resolution in Makefile.dep, as
+# feature resolution depends on the used modules. As these modules however have
+# no dependencies(except for periph_common), no dependency resolution is needed.
+
+PERIPH_FEATURES := $(filter periph_%,$(FEATURES_USED))
+# all periph features correspond to a periph submodule
+# FEATURES_USED is defined in Makefile.features
+USEMODULE += $(PERIPH_FEATURES)
+
+# Add all USED periph_% init modules unless they are blacklisted
+ifneq (,$(filter periph_init, $(USEMODULE)))
+  PERIPH_MODULES := $(filter-out periph_init% periph_common,\
+                                 $(filter periph_%,$(USEMODULE)))
+  # Use simple expansion to avoid USEMODULE referencing itself
+  PERIPH_INIT_MODULES := $(subst periph_,periph_init_,$(PERIPH_MODULES))
+  DEFAULT_MODULE += $(PERIPH_INIT_MODULES)
+endif
+
+# select cpu_check_address pseudomodule if the corresponding feature is used
+USEMODULE += $(filter cpu_check_address, $(FEATURES_USED))
+
+# include periph_common if any periph_* driver is used
+ifneq (,$(filter periph_%, $(USEMODULE)))
+  USEMODULE += periph_common
+endif

makefiles/info-global.inc.mk

@@ -1,6 +1,16 @@
 .PHONY: info-buildsizes info-buildsizes-diff info-features-missing \
         info-boards-features-missing
 
+# Perform dependency resolution without having included
+# $(RIOTBASE)/Makefile.features for now. This results in no optional modules and
+# no architecture specific handling being done. The result will be a subset of
+# used modules and requirements that is present in for *all* boards, so this can
+# be cached to speed up individual dependency checks
+include $(RIOTMAKE)/defaultmodules.inc.mk
+# add default modules
+USEMODULE += $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE))
+include $(RIOTMAKE)/dependency_resolution.inc.mk
+
 BOARDSDIR_GLOBAL := $(BOARDSDIR)
 USEMODULE_GLOBAL := $(USEMODULE)
 USEPKG_GLOBAL := $(USEPKG)
@@ -27,6 +37,7 @@ define board_unsatisfied_features
 
   # Remove board specific variables set by Makefile.features/Makefile.dep
   FEATURES_PROVIDED :=
+  FEATURES_USED :=
 
   # Undefine variables that must not be defined when starting.
   # Some are sometime set as `?=`
@@ -40,20 +51,28 @@ define board_unsatisfied_features
   endif
 
   include $(RIOTBASE)/Makefile.features
-
-  include $(RIOTMAKE)/defaultmodules.inc.mk
-  USEMODULE += $(filter-out $(DISABLE_MODULE), $(DEFAULT_MODULE))
-
-  include $(RIOTBASE)/Makefile.dep
-
+  # FEATURES_USED must be populated first in this case so that dependency
+  # resolution can take optional features into account during the first pass.
+  # Also: This allows us to skip resolution if already a missing feature is
+  # detected
+  include $(RIOTMAKE)/features_check.inc.mk
   ifneq (,$$(FEATURES_MISSING))
+    # Skip full dependency resolution, as even without optional modules features
+    # and architecture specific limitations already some features are missing
     BOARDS_FEATURES_MISSING += "$(1) $$(FEATURES_MISSING)"
     BOARDS_WITH_MISSING_FEATURES += $(1)
-  endif
-
-  ifneq (,$$(FEATURES_USED_BLACKLISTED))
-    BOARDS_FEATURES_USED_BLACKLISTED += "$(1) $$(FEATURES_USED_BLACKLISTED)"
-    BOARDS_WITH_BLACKLISTED_FEATURES += $(1)
+  else
+    include $(RIOTMAKE)/dependency_resolution.inc.mk
+
+    ifneq (,$$(FEATURES_MISSING))
+      BOARDS_FEATURES_MISSING += "$(1) $$(FEATURES_MISSING)"
+      BOARDS_WITH_MISSING_FEATURES += $(1)
+    endif
+
+    ifneq (,$$(FEATURES_USED_BLACKLISTED))
+      BOARDS_FEATURES_USED_BLACKLISTED += "$(1) $$(FEATURES_USED_BLACKLISTED)"
+      BOARDS_WITH_BLACKLISTED_FEATURES += $(1)
+    endif
   endif
 
   ifneq (,$$(DEPENDENCY_DEBUG))