From 17479f77ccc703048690961fa9b6795bb9c5171b Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 5 May 2026 14:16:50 -0700 Subject: [PATCH 1/2] [pants_ng] Scaffolding for a pants_ng mode. In this mode the command line is parsed as an NG invocation, and dispatched appropriately. Of course at the moment there are no implementations to dispatch to. That will follow. This does expose a new option, `pants_ng` to users. There is a big warning not to set it, but we're not trying to hide that we're working on a new thing, so I am comfortable with this. --- src/python/pants/bin/daemon_pants_runner.py | 25 +++++++-- src/python/pants/bin/local_pants_runner.py | 54 ++++++++++++------- src/python/pants/bin/pants_loader.py | 19 +++++++ src/python/pants/bin/pants_runner.py | 26 +++++++-- .../pants/build_graph/build_configuration.py | 11 ++++ src/python/pants/init/engine_initializer.py | 45 +++++++++++++--- src/python/pants/init/extension_loader.py | 10 ++-- .../pants/init/extension_loader_test.py | 2 +- src/python/pants/init/options_initializer.py | 1 + src/python/pants/init/specs_calculator.py | 5 +- src/python/pants/option/bootstrap_options.py | 13 +++++ 11 files changed, 169 insertions(+), 42 deletions(-) diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py index 48a455b36d3..f6914a79659 100644 --- a/src/python/pants/bin/daemon_pants_runner.py +++ b/src/python/pants/bin/daemon_pants_runner.py @@ -11,7 +11,7 @@ from pants.base.exiter import PANTS_FAILED_EXIT_CODE, ExitCode from pants.bin.local_pants_runner import LocalPantsRunner from pants.engine.env_vars import CompleteEnvironmentVars -from pants.engine.internals.native_engine import PySessionCancellationLatch +from pants.engine.internals.native_engine import PyNgInvocation, PySessionCancellationLatch from pants.init.logging import stdio_destination from pants.option.options_bootstrapper import OptionsBootstrapper from pants.pantsd.pants_daemon_core import PantsDaemonCore @@ -107,6 +107,10 @@ def single_daemonized_run( method should not need any special handling for the fact that it's running in a proxied environment. """ + # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from + # config and env. We can't parse args until we know if we're ng or og. + options_bootstrapper = OptionsBootstrapper.create(args=[], env=env, allow_pantsrc=True) + pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng try: logger.debug("Connected to pantsd") @@ -123,9 +127,21 @@ def single_daemonized_run( ) start_time = float(env_start_time) if env_start_time else time.time() - options_bootstrapper = OptionsBootstrapper.create( - args=args, env=env, allow_pantsrc=True - ) + if pants_ng: + logger.info("DaemonPantsRunner running as pants_ng") + ng_invocation = PyNgInvocation.from_args(args[1:]) + # Allow the existing logic to read flags in global position (note, these can be + # prefixed with a scope so they are not necessarily just global options), so + # that the engine can configure itself. + global_flags = ng_invocation.global_flag_strings() + options_bootstrapper = OptionsBootstrapper.create( + args=[args[0], *global_flags], env=env, allow_pantsrc=True + ) + else: + ng_invocation = None + options_bootstrapper = OptionsBootstrapper.create( + args=args, env=env, allow_pantsrc=True + ) # Run using the pre-warmed Session. complete_env = CompleteEnvironmentVars(env) @@ -137,6 +153,7 @@ def single_daemonized_run( scheduler=scheduler, options_initializer=options_initializer, cancellation_latch=cancellation_latch, + ng_invocation=ng_invocation, ) return runner.run(start_time) except Exception as e: diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index cb185717c0e..3fa13bbc569 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -10,13 +10,17 @@ from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode from pants.base.specs import Specs -from pants.base.specs_parser import SpecsParser from pants.build_graph.build_configuration import BuildConfiguration from pants.core.environments.rules import determine_bootstrap_environment from pants.engine.env_vars import CompleteEnvironmentVars from pants.engine.goal import CurrentExecutingGoals from pants.engine.internals import native_engine -from pants.engine.internals.native_engine import PyExecutor, PySessionCancellationLatch +from pants.engine.internals.native_engine import ( + PyExecutor, + PyNgInvocation, + PyNgOptions, + PySessionCancellationLatch, +) from pants.engine.internals.scheduler import ExecutionError from pants.engine.internals.selectors import Params from pants.engine.internals.session import SessionValues @@ -63,6 +67,7 @@ class LocalPantsRunner: union_membership: UnionMembership is_pantsd_run: bool working_dir: str + ng_invocation: PyNgInvocation | None @classmethod def create( @@ -73,6 +78,7 @@ def create( options_initializer: OptionsInitializer | None = None, scheduler: GraphScheduler | None = None, cancellation_latch: PySessionCancellationLatch | None = None, + ng_invocation: PyNgInvocation | None = None, ) -> LocalPantsRunner: """Creates a new LocalPantsRunner instance by parsing options. @@ -126,8 +132,18 @@ def create( ) with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True): global_options = options.for_global_scope() + session_values_dict = { + OptionsBootstrapper: options_bootstrapper, + CompleteEnvironmentVars: env, + CurrentExecutingGoals: CurrentExecutingGoals(), + } + if ng_invocation is not None: + session_values_dict[PyNgInvocation] = ng_invocation + session_values_dict[PyNgOptions] = PyNgOptions( + ng_invocation, dict(env.items()), include_derivation=False + ) graph_session = scheduler.new_session( - run_tracker.run_id, + build_id=run_tracker.run_id, dynamic_ui=global_options.dynamic_ui, ui_use_prodash=global_options.dynamic_ui_renderer == DynamicUIRenderer.experimental_prodash, @@ -140,17 +156,17 @@ def create( for level in global_options.log_levels_by_target.values() ), ), - session_values=SessionValues( - { - OptionsBootstrapper: options_bootstrapper, - CompleteEnvironmentVars: env, - CurrentExecutingGoals: CurrentExecutingGoals(), - } - ), + session_values=SessionValues(session_values_dict), cancellation_latch=cancellation_latch, ) + if ng_invocation: + specs_strs = ng_invocation.specs() + else: + specs_strs = tuple(options.specs) + specs = calculate_specs( + specs_strs=specs_strs, options_bootstrapper=options_bootstrapper, options=options, session=graph_session.scheduler_session, @@ -169,6 +185,7 @@ def create( union_membership=union_membership, is_pantsd_run=is_pantsd_run, working_dir=working_dir, + ng_invocation=ng_invocation, ) def _perform_run(self, goals: tuple[str, ...]) -> ExitCode: @@ -251,10 +268,12 @@ def _run_auxiliary_goal(context: AuxiliaryGoalContext, goal: Any) -> ExitCode: ) def _run_inner(self) -> ExitCode: - if self.options.builtin_or_auxiliary_goal: + if self.ng_invocation: + goals = self.ng_invocation.goals() + elif self.options.builtin_or_auxiliary_goal: return self._run_builtin_or_auxiliary_goal(self.options.builtin_or_auxiliary_goal) - - goals = tuple(self.options.goals) + else: + goals = tuple(self.options.goals) if not goals: return PANTS_SUCCEEDED_EXIT_CODE @@ -268,13 +287,8 @@ def _run_inner(self) -> ExitCode: return PANTS_FAILED_EXIT_CODE def run(self, start_time: float) -> ExitCode: - spec_parser = SpecsParser(working_dir=self.working_dir) - specs = [] - for spec_str in self.options.specs: - spec, is_ignore = spec_parser.parse_spec(spec_str) - specs.append(f"-{spec}" if is_ignore else str(spec)) - - self.run_tracker.start(run_start_time=start_time, specs=specs) + specs_strs = list(self.ng_invocation.specs()) if self.ng_invocation else self.options.specs + self.run_tracker.start(run_start_time=start_time, specs=specs_strs) global_options = self.options.for_global_scope() streaming_reporter = StreamingWorkunitHandler( diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 1b30330db59..cdf89bf4e81 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -4,10 +4,12 @@ import importlib import locale +import logging import os import sys import time import warnings +from datetime import datetime from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( @@ -19,6 +21,9 @@ from pants.engine.internals import native_engine from pants.util.strutil import softwrap +# pants: infer-dep(/src/python/pants/backend/**/register.py) +# pants: infer-dep(/src/python/pants/core/**/*) + class PantsLoader: """Initial entrypoint for pants. @@ -118,6 +123,20 @@ def run_default_entrypoint() -> None: @classmethod def main(cls) -> None: + # Set up logging. This is just temporary, as the PantsRunner will re-initialize logging to + # be emitted via Rust, but until that happens we want basic logging to do something useful. + # We set it up to be visually compatible with the Rust logging. + class HundredthsFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + dt = datetime.fromtimestamp(record.created) + formatted_time = dt.strftime(datefmt or "%H:%M:%S") + hundredths = str(dt.microsecond // 10000).zfill(2) + return f"{formatted_time}.{hundredths}" + + handler = logging.StreamHandler() + handler.setFormatter(HundredthsFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s")) + logging.basicConfig(level=logging.INFO, handlers=[handler]) + native_engine.initialize() cls.setup_warnings() cls.ensure_locale() diff --git a/src/python/pants/bin/pants_runner.py b/src/python/pants/bin/pants_runner.py index 4b956cb5395..5d7cd94b056 100644 --- a/src/python/pants/bin/pants_runner.py +++ b/src/python/pants/bin/pants_runner.py @@ -15,6 +15,7 @@ from pants.base.exception_sink import ExceptionSink from pants.base.exiter import ExitCode from pants.engine.env_vars import CompleteEnvironmentVars +from pants.engine.internals.native_engine import PyNgInvocation from pants.init.logging import initialize_stdio, stdio_destination from pants.init.util import init_workdir from pants.option.bootstrap_options import BootstrapOptions @@ -74,9 +75,27 @@ def scrub_pythonpath() -> None: def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() - options_bootstrapper = OptionsBootstrapper.create( - args=self.args, env=self.env, allow_pantsrc=True - ) + # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from + # config and env. We can't parse args until we know if we're ng or og. + options_bootstrapper = OptionsBootstrapper.create(args=[], env=self.env, allow_pantsrc=True) + pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng + + if pants_ng: + logger.info("PantsRunner running as pants_ng") + ng_invocation = PyNgInvocation.from_args(tuple(self.args[1:])) + # Allow the existing logic to read flags in global position (note, these can be + # prefixed with a scope so they are not necessarily just global options), so + # that the engine can configure itself. + global_flags = ng_invocation.global_flag_strings() + options_bootstrapper = OptionsBootstrapper.create( + args=[self.args[0], *global_flags], env=self.env, allow_pantsrc=True + ) + else: + ng_invocation = None + options_bootstrapper = OptionsBootstrapper.create( + args=self.args, env=self.env, allow_pantsrc=True + ) + with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() @@ -155,6 +174,7 @@ def run(self, start_time: float) -> ExitCode: env=CompleteEnvironmentVars(self.env), working_dir=os.getcwd(), options_bootstrapper=options_bootstrapper, + ng_invocation=ng_invocation, ) return runner.run(start_time) diff --git a/src/python/pants/build_graph/build_configuration.py b/src/python/pants/build_graph/build_configuration.py index 6d6227ed76b..19fa9ee007c 100644 --- a/src/python/pants/build_graph/build_configuration.py +++ b/src/python/pants/build_graph/build_configuration.py @@ -17,6 +17,7 @@ from pants.engine.rules import Rule, RuleIndex, collect_rules, rule from pants.engine.target import Target from pants.engine.unions import UnionRule +from pants.ng.goal import GoalSubsystemNg from pants.option.alias import CliOptions from pants.option.global_options import GlobalOptions from pants.option.scope import OptionsParsingSettings, ScopeInfo, normalize_scope @@ -143,6 +144,7 @@ class Builder: ) _allow_unknown_options: bool = False _remote_auth_plugin: Callable | None = None + _pants_ng: bool = False def registered_aliases(self) -> BuildFileAliases: """Return the registered aliases exposed in BUILD files. @@ -217,6 +219,15 @@ def register_subsystems( ) for subsystem in subsystems: + if ( + issubclass(subsystem, GoalSubsystem) + and issubclass(subsystem, GoalSubsystemNg) != self._pants_ng + ): + # Don't register goal subsystems that aren't pertinent to our og/ng state. + # This allows GoalSubsystemNgs to use the same scopes as og GoalSubsystems + # without collision. Note that the @rules that consume these subsystems + # can still be registered, but the subsystems cannot be configured. + continue self._subsystem_to_providers[subsystem].append(plugin_or_backend) def register_rules(self, plugin_or_backend: str, rules: Iterable[Rule | UnionRule]): diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index ff53c581fb7..26ffb69f70e 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -37,12 +37,13 @@ from pants.engine.internals.scheduler import Scheduler, SchedulerSession from pants.engine.internals.selectors import Params from pants.engine.internals.session import SessionValues -from pants.engine.rules import QueryRule, collect_rules, rule +from pants.engine.rules import QueryRule, Rule, collect_rules, rule from pants.engine.streaming_workunit_handler import rules as streaming_workunit_handler_rules from pants.engine.target import RegisteredTargetTypes from pants.engine.unions import UnionMembership, UnionRule from pants.init import specs_calculator from pants.init.bootstrap_scheduler import BootstrapStatus +from pants.ng.goal import GoalNg, GoalSubsystemNg from pants.option.bootstrap_options import ( DEFAULT_EXECUTION_OPTIONS, DynamicRemoteOptions, @@ -141,8 +142,13 @@ def run_goal_rules( env_name = determine_bootstrap_environment(self.scheduler_session) for goal in goals: - goal_product = self.goal_map[goal] - if not goal_product.subsystem_cls.activated(union_membership): + goal_product = self.goal_map.get(goal) + if goal_product is None: + # This must be an ng command, since the og goals have already + # been validated by the options parser. + raise UnknownCommand(goal) + pants_ng = issubclass(goal_product.subsystem_cls, GoalSubsystemNg) + if not pants_ng and not goal_product.subsystem_cls.activated(union_membership): raise GoalNotActivatedException(goal) # NB: Keep this in sync with the property `goal_param_types`. params = Params(specs, self.console, workspace, env_name) @@ -167,15 +173,21 @@ class GoalMappingError(Exception): """Raised when a goal cannot be mapped to an @rule.""" @staticmethod - def _make_goal_map_from_rules(rules) -> Mapping[str, type[Goal]]: + def _make_goal_map_from_rules(rules, pants_ng: bool) -> Mapping[str, type[Goal]]: goal_map: dict[str, type[Goal]] = {} for r in rules: output_type = getattr(r, "output_type", None) - if not output_type or not issubclass(output_type, Goal): + if ( + not output_type + or not issubclass(output_type, Goal) + or issubclass(output_type, GoalNg) != pants_ng + ): continue goal = r.output_type.name - deprecated_goal = r.output_type.subsystem_cls.deprecated_options_scope + deprecated_goal = ( + None if pants_ng else r.output_type.subsystem_cls.deprecated_options_scope + ) for goal_name in [goal, deprecated_goal] if deprecated_goal else [goal]: if goal_name in goal_map: raise EngineInitializer.GoalMappingError( @@ -220,6 +232,7 @@ def setup_graph( engine_visualize_to=bootstrap_options.engine_visualize_to, watch_filesystem=bootstrap_options.watch_filesystem, is_bootstrap=is_bootstrap, + pants_ng=bootstrap_options.pants_ng, ) @staticmethod @@ -240,6 +253,7 @@ def setup_graph_extended( engine_visualize_to: str | None = None, watch_filesystem: bool = True, is_bootstrap: bool = False, + pants_ng: bool = False, ) -> GraphScheduler: build_root_path = build_root or get_buildroot() @@ -309,7 +323,19 @@ async def current_executing_goals(session_values: SessionValues) -> CurrentExecu ) ) - goal_map = EngineInitializer._make_goal_map_from_rules(rules) + def is_pertinent_rule(rule: Rule | UnionRule, ng: bool) -> bool: + output_type = getattr(rule, "output_type", None) + if not output_type: # This is a UnionRule, so keep it. + return True + if issubclass(output_type, Goal) and issubclass(output_type, GoalNg) != ng: + return False + return True + + # Strip out all the goal_rules that don't pertain to our current ng/og state. This allows + # ng commands to use the same names as og goals, without collision. + rules = FrozenOrderedSet(rule for rule in rules if is_pertinent_rule(rule, pants_ng)) + + goal_map = EngineInitializer._make_goal_map_from_rules(rules, pants_ng) union_membership = UnionMembership.from_rules( ( @@ -370,6 +396,11 @@ def ensure_optional_absolute_path(v: str | None) -> str | None: return GraphScheduler(scheduler, goal_map) +class UnknownCommand(Exception): + def __init__(self, goal: str) -> None: + super().__init__(f"Unknown command `{goal.replace('.', ' ')}`") + + class GoalNotActivatedException(Exception): def __init__(self, goal_name: str) -> None: super().__init__( diff --git a/src/python/pants/init/extension_loader.py b/src/python/pants/init/extension_loader.py index 986c1676ffd..f24b9be1a03 100644 --- a/src/python/pants/init/extension_loader.py +++ b/src/python/pants/init/extension_loader.py @@ -34,18 +34,18 @@ class PluginLoadOrderError(PluginLoadingError): def load_backends_and_plugins( plugins: list[str], backends: list[str], - bc_builder: BuildConfiguration.Builder | None = None, + bc_builder: BuildConfiguration.Builder, ) -> BuildConfiguration: """Load named plugins and source backends. - :param plugins: v2 plugins to load. - :param backends: v2 backends to load. + :param plugins: plugins to load. + :param backends: backends to load. :param bc_builder: The BuildConfiguration (for adding aliases). """ - bc_builder = bc_builder or BuildConfiguration.Builder() load_build_configuration_from_source(bc_builder, backends) load_plugins(bc_builder, plugins) - register_builtin_goals(bc_builder) + if not bc_builder._pants_ng: + register_builtin_goals(bc_builder) return bc_builder.create() diff --git a/src/python/pants/init/extension_loader_test.py b/src/python/pants/init/extension_loader_test.py index 20950816681..8f446648805 100644 --- a/src/python/pants/init/extension_loader_test.py +++ b/src/python/pants/init/extension_loader_test.py @@ -359,7 +359,7 @@ def reg_alias(): with self.create_register(build_file_aliases=lambda: aliases) as backend_module: backends = [backend_module] build_configuration = load_backends_and_plugins( - plugins, backends, bc_builder=self.bc_builder + plugins, backends, pants_ng=False, bc_builder=self.bc_builder ) # The backend should load first, then the plugins, therefore the alias registered in # the plugin will override the alias registered by the backend diff --git a/src/python/pants/init/options_initializer.py b/src/python/pants/init/options_initializer.py index 0c9830abd0a..b92ee353dd7 100644 --- a/src/python/pants/init/options_initializer.py +++ b/src/python/pants/init/options_initializer.py @@ -61,6 +61,7 @@ def _initialize_build_configuration( return load_backends_and_plugins( bootstrap_options.plugins, bootstrap_options.backend_packages, + BuildConfiguration.Builder(_pants_ng=bootstrap_options.pants_ng), ) diff --git a/src/python/pants/init/specs_calculator.py b/src/python/pants/init/specs_calculator.py index 4d1df82c830..53d5ae14fad 100644 --- a/src/python/pants/init/specs_calculator.py +++ b/src/python/pants/init/specs_calculator.py @@ -2,7 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -from typing import cast +from typing import Iterable, cast from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.base.specs import AddressLiteralSpec, FileLiteralSpec, RawSpecs, Specs @@ -31,6 +31,7 @@ class InvalidSpecConstraint(Exception): def calculate_specs( + specs_strs: Iterable[str], options_bootstrapper: OptionsBootstrapper, options: Options, session: SchedulerSession, @@ -40,7 +41,7 @@ def calculate_specs( global_options = options.for_global_scope() unmatched_cli_globs = global_options.unmatched_cli_globs specs = SpecsParser(working_dir=working_dir).parse_specs( - options.specs, + specs_strs, description_of_origin="CLI arguments", unmatched_glob_behavior=unmatched_cli_globs, ) diff --git a/src/python/pants/option/bootstrap_options.py b/src/python/pants/option/bootstrap_options.py index a6faebf3778..75894f90ec2 100644 --- a/src/python/pants/option/bootstrap_options.py +++ b/src/python/pants/option/bootstrap_options.py @@ -1781,6 +1781,19 @@ def file_downloads_max_attempts(self) -> int: ), ) + pants_ng = BoolOption( + default=False, + advanced=True, + help=softwrap( + """ + Enable the Pants "next generation" mode. + + Note that this is currently under development and is *not* yet useful to end users. + Do not set this unless you truly know what you're doing. + """ + ), + ) + @classmethod def validate_instance(cls, opts: OptionValueContainer): """Validates an instance of global options for cases that are not prohibited via From c82d0f333c934c193bc0095cc0321e2179b50790 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 5 May 2026 15:18:25 -0700 Subject: [PATCH 2/2] Fix test --- src/python/pants/init/extension_loader_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/init/extension_loader_test.py b/src/python/pants/init/extension_loader_test.py index 8f446648805..20950816681 100644 --- a/src/python/pants/init/extension_loader_test.py +++ b/src/python/pants/init/extension_loader_test.py @@ -359,7 +359,7 @@ def reg_alias(): with self.create_register(build_file_aliases=lambda: aliases) as backend_module: backends = [backend_module] build_configuration = load_backends_and_plugins( - plugins, backends, pants_ng=False, bc_builder=self.bc_builder + plugins, backends, bc_builder=self.bc_builder ) # The backend should load first, then the plugins, therefore the alias registered in # the plugin will override the alias registered by the backend