Skip to content

build-system: auto-locate modules by name #16465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
maribu wants to merge 4 commits into from
Closed

build-system: auto-locate modules by name #16465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4b683be
sys/bus: move to sys/sys_bus
maribu May 10, 2021
1769b6f
build-system: auto-locate modules by name
maribu May 10, 2021
74bd5af
fixup! build-system: auto-locate modules by name
maribu May 11, 2021
3d2fca2
fixup! build-system: auto-locate modules by name
maribu May 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Makefile.include
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ RIOTCPU ?= $(RIOTBASE)/cpu
# Deprecated to set RIOTBOARD, use EXTERNAL_BOARD_DIRS
RIOTBOARD ?= $(RIOTBASE)/boards
EXTERNAL_BOARD_DIRS ?=
MODULE_DIRS := $(RIOTBASE)/drivers
MODULE_DIRS += $(RIOTBASE)/sys
RIOTMAKE ?= $(RIOTBASE)/makefiles
RIOTPKG ?= $(RIOTBASE)/pkg
RIOTTOOLS ?= $(RIOTBASE)/dist/tools
Expand Down Expand Up @@ -149,6 +151,7 @@ EXTERNAL_BOARD_DIRS := $(foreach dir,\
EXTERNAL_MODULE_DIRS := $(foreach dir,\
$(EXTERNAL_MODULE_DIRS),\
$(abspath $(dir)))
MODULE_DIRS := $(foreach dir,$(MODULE_DIRS),$(abspath $(dir)))

# Ensure that all directories are set and don't contain spaces.
ifneq (, $(filter-out 1, $(foreach v,$(__DIRECTORY_VARIABLES),$(words $($(v))))))
Expand Down Expand Up @@ -403,6 +406,8 @@ ifeq (1,$(TEST_KCONFIG))
USEMODULE := $(KCONFIG_MODULES)
KCONFIG_PACKAGES := $(call lowercase,$(patsubst CONFIG_PACKAGE_%,%,$(filter CONFIG_PACKAGE_%,$(.VARIABLES))))
USEPKG := $(KCONFIG_PACKAGES)
# auto-locate modules
include $(RIOTMAKE)/locate_modules.inc.mk
else
# check if required features are provided and update $(FEATURES_USED)
include $(RIOTMAKE)/features_check.inc.mk
Expand Down Expand Up @@ -619,8 +624,8 @@ endif
# We assume $(LINK) to be gcc-like. Use `LINKFLAGPREFIX :=` for ld-like linker options.
LINKFLAGPREFIX ?= -Wl,

# Also build external modules
DIRS += $(EXTERNAL_MODULE_PATHS)
# Also build automatically located modules
DIRS += $(EXTERNAL_MODULE_PATHS) $(MODULE_PATHS)

# Define dependencies required for building (headers, downloading source files,)
BUILDDEPS += $(RIOTBUILD_CONFIG_HEADER_C)
Expand Down
2 changes: 0 additions & 2 deletions drivers/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
DIRS += $(dir $(wildcard $(addsuffix /Makefile, $(USEMODULE))))

ifneq (,$(filter saul_init_devs,$(USEMODULE)))
DIRS += saul/init_devs
endif
Expand Down
5 changes: 2 additions & 3 deletions makefiles/dependency_resolution.inc.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
# until no new modules, pkgs, or features are pull in order to catch all
# transient dependencies

# Locate used modules in $(EXTERNAL_MODULE_DIRS).
EXTERNAL_MODULE_PATHS := $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\
$(foreach mod,$(USEMODULE),$(dir $(wildcard $(dir)/$(mod)/Makefile)))))
# auto-locate modules
include $(RIOTMAKE)/locate_modules.inc.mk

# Back up current state to detect changes
OLD_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
Expand Down
11 changes: 11 additions & 0 deletions makefiles/locate_modules.inc.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Only locate non-pseudo-modules
MODS2LOCATE := $(filter-out $(PSEUDOMODULES) $(NO_AUTOLOCATE),\

@fjmolinas fjmolinas Oct 20, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is mods2?

@maribu maribu Oct 20, 2021

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

modules to locate ;-)

$(USEMODULE)) $(filter $(NO_PSEUDOMODULES),$(USEMODULE))
EXTERNAL_MODULE_PATHS := $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\
$(foreach mod,$(MODS2LOCATE),$(dir $(wildcard $(dir)/$(mod)/Makefile)))))

# Ignore modules already located as external modules
MODS2LOCATE := $(filter-out $(basename $(EXTERNAL_MODULE_PATHS)),$(MODS2LOCATE))

MODULE_PATHS := $(sort $(foreach dir,$(MODULE_DIRS),\
$(foreach mod,$(MODS2LOCATE),$(dir $(wildcard $(dir)/$(mod)/Makefile)))))
29 changes: 0 additions & 29 deletions sys/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,6 @@ endif
ifneq (,$(filter cord_lc,$(USEMODULE)))
DIRS += net/application_layer/cord/lc
endif
ifneq (,$(filter cpp11-compat,$(USEMODULE)))
DIRS += cpp11-compat
endif
ifneq (,$(filter credman,$(USEMODULE)))
DIRS += net/credman
endif
Expand All @@ -41,9 +38,6 @@ endif
ifneq (,$(filter dummy_thread,$(USEMODULE)))
DIRS += test_utils/dummy_thread
endif
ifneq (,$(filter eepreg,$(USEMODULE)))
DIRS += eepreg
endif
ifneq (,$(filter emcute,$(USEMODULE)))
DIRS += net/application_layer/emcute
endif
Expand Down Expand Up @@ -89,9 +83,6 @@ endif
ifneq (,$(filter l2util,$(USEMODULE)))
DIRS += net/link_layer/l2util
endif
ifneq (,$(filter log_%,$(USEMODULE)))
DIRS += log
endif
ifneq (,$(filter nanocoap,$(USEMODULE)))
DIRS += net/application_layer/nanocoap
endif
Expand All @@ -110,12 +101,6 @@ endif
ifneq (,$(filter netstats_neighbor,$(USEMODULE)))
DIRS += net/netstats
endif
ifneq (,$(filter sema,$(USEMODULE)))
DIRS += sema
endif
ifneq (,$(filter sema_inv,$(USEMODULE)))
DIRS += sema_inv
endif
ifneq (,$(filter sixlowpan,$(USEMODULE)))
DIRS += net/network_layer/sixlowpan
endif
Expand All @@ -131,9 +116,6 @@ endif
ifneq (,$(filter sock_util,$(USEMODULE)))
DIRS += net/sock
endif
ifneq (,$(filter oneway_malloc,$(USEMODULE)))
DIRS += oneway-malloc
endif
ifneq (,$(filter posix_inet,$(USEMODULE)))
DIRS += posix/inet
endif
Expand Down Expand Up @@ -161,12 +143,6 @@ endif
ifneq (,$(filter shell_commands,$(USEMODULE)))
DIRS += shell/commands
endif
ifneq (,$(filter suit%,$(USEMODULE)))
DIRS += suit
endif
ifneq (,$(filter sys_bus,$(USEMODULE)))
DIRS += bus
endif
ifneq (,$(filter tcp,$(USEMODULE)))
DIRS += net/transport_layer/tcp
endif
Expand All @@ -188,10 +164,5 @@ endif
ifneq (,$(filter ztimer_core,$(USEMODULE)))
DIRS += ztimer
endif
ifneq (,$(filter ztimer_xtimer_compat,$(USEMODULE)))
FILTER += xtimer
endif

DIRS += $(dir $(wildcard $(addsuffix /Makefile, $(filter-out $(FILTER), $(USEMODULE)))))

include $(RIOTBASE)/Makefile.base
1 change: 1 addition & 0 deletions sys/Makefile.dep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,7 @@ ifneq (,$(filter xtimer,$(USEMODULE)))
endif
else
# ztimer_xtimer_compat is used, all of *xtimer's API will be mapped on ztimer.*
NO_AUTOLOCATE += xtimer

@fjmolinas fjmolinas Jun 8, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this be affected by the order of inclusion?

@maribu maribu Jun 8, 2021

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The location of modules is done fully again at each recursion step. If xtimer would be auto-located in a transient state, it would be dropped later on. But some care should be taken so that an Makefile.dep doesn't do anything xtimer related when ztimer_xtimer_compat is also used. But since there is no Makefile.dep in sys/xtimer and in sys/Makefile.dep (above code) the correct guards are in place, this should just work.

A bit of a foot gun, but choosing different implementation options for the same API is just not properly modeled in the build system, so some friction is expected.

endif
endif

Expand Down
2 changes: 0 additions & 2 deletions sys/bus/Makefile → sys/sys_bus/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
MODULE = sys_bus

include $(RIOTBASE)/Makefile.base
0 sys/bus/sys_bus_init.c → sys/sys_bus/sys_bus_init.c
View file
Open in desktop
File renamed without changes.