diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py index 48a455b36d3..f6914a79659 100644 --- a/src/python/pants/bin/daemon_pants_runner.py +++ b/src/python/pants/bin/daemon_pants_runner.py @@ -11,7 +11,7 @@ from pants.base.exiter import PANTS_FAILED_EXIT_CODE, ExitCode from pants.bin.local_pants_runner import LocalPantsRunner from pants.engine.env_vars import CompleteEnvironmentVars -from pants.engine.internals.native_engine import PySessionCancellationLatch +from pants.engine.internals.native_engine import PyNgInvocation, PySessionCancellationLatch from pants.init.logging import stdio_destination from pants.option.options_bootstrapper import OptionsBootstrapper from pants.pantsd.pants_daemon_core import PantsDaemonCore @@ -107,6 +107,10 @@ def single_daemonized_run( method should not need any special handling for the fact that it's running in a proxied environment. """ + # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from + # config and env. We can't parse args until we know if we're ng or og. + options_bootstrapper = OptionsBootstrapper.create(args=[], env=env, allow_pantsrc=True) + pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng try: logger.debug("Connected to pantsd") @@ -123,9 +127,21 @@ def single_daemonized_run( ) start_time = float(env_start_time) if env_start_time else time.time() - options_bootstrapper = OptionsBootstrapper.create( - args=args, env=env, allow_pantsrc=True - ) + if pants_ng: + logger.info("DaemonPantsRunner running as pants_ng") + ng_invocation = PyNgInvocation.from_args(args[1:]) + # Allow the existing logic to read flags in global position (note, these can be + # prefixed with a scope so they are not necessarily just global options), so + # that the engine can configure itself. + global_flags = ng_invocation.global_flag_strings() + options_bootstrapper = OptionsBootstrapper.create( + args=[args[0], *global_flags], env=env, allow_pantsrc=True + ) + else: + ng_invocation = None + options_bootstrapper = OptionsBootstrapper.create( + args=args, env=env, allow_pantsrc=True + ) # Run using the pre-warmed Session. complete_env = CompleteEnvironmentVars(env) @@ -137,6 +153,7 @@ def single_daemonized_run( scheduler=scheduler, options_initializer=options_initializer, cancellation_latch=cancellation_latch, + ng_invocation=ng_invocation, ) return runner.run(start_time) except Exception as e: diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index cb185717c0e..3fa13bbc569 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -10,13 +10,17 @@ from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode from pants.base.specs import Specs -from pants.base.specs_parser import SpecsParser from pants.build_graph.build_configuration import BuildConfiguration from pants.core.environments.rules import determine_bootstrap_environment from pants.engine.env_vars import CompleteEnvironmentVars from pants.engine.goal import CurrentExecutingGoals from pants.engine.internals import native_engine -from pants.engine.internals.native_engine import PyExecutor, PySessionCancellationLatch +from pants.engine.internals.native_engine import ( + PyExecutor, + PyNgInvocation, + PyNgOptions, + PySessionCancellationLatch, +) from pants.engine.internals.scheduler import ExecutionError from pants.engine.internals.selectors import Params from pants.engine.internals.session import SessionValues @@ -63,6 +67,7 @@ class LocalPantsRunner: union_membership: UnionMembership is_pantsd_run: bool working_dir: str + ng_invocation: PyNgInvocation | None @classmethod def create( @@ -73,6 +78,7 @@ def create( options_initializer: OptionsInitializer | None = None, scheduler: GraphScheduler | None = None, cancellation_latch: PySessionCancellationLatch | None = None, + ng_invocation: PyNgInvocation | None = None, ) -> LocalPantsRunner: """Creates a new LocalPantsRunner instance by parsing options. @@ -126,8 +132,18 @@ def create( ) with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True): global_options = options.for_global_scope() + session_values_dict = { + OptionsBootstrapper: options_bootstrapper, + CompleteEnvironmentVars: env, + CurrentExecutingGoals: CurrentExecutingGoals(), + } + if ng_invocation is not None: + session_values_dict[PyNgInvocation] = ng_invocation + session_values_dict[PyNgOptions] = PyNgOptions( + ng_invocation, dict(env.items()), include_derivation=False + ) graph_session = scheduler.new_session( - run_tracker.run_id, + build_id=run_tracker.run_id, dynamic_ui=global_options.dynamic_ui, ui_use_prodash=global_options.dynamic_ui_renderer == DynamicUIRenderer.experimental_prodash, @@ -140,17 +156,17 @@ def create( for level in global_options.log_levels_by_target.values() ), ), - session_values=SessionValues( - { - OptionsBootstrapper: options_bootstrapper, - CompleteEnvironmentVars: env, - CurrentExecutingGoals: CurrentExecutingGoals(), - } - ), + session_values=SessionValues(session_values_dict), cancellation_latch=cancellation_latch, ) + if ng_invocation: + specs_strs = ng_invocation.specs() + else: + specs_strs = tuple(options.specs) + specs = calculate_specs( + specs_strs=specs_strs, options_bootstrapper=options_bootstrapper, options=options, session=graph_session.scheduler_session, @@ -169,6 +185,7 @@ def create( union_membership=union_membership, is_pantsd_run=is_pantsd_run, working_dir=working_dir, + ng_invocation=ng_invocation, ) def _perform_run(self, goals: tuple[str, ...]) -> ExitCode: @@ -251,10 +268,12 @@ def _run_auxiliary_goal(context: AuxiliaryGoalContext, goal: Any) -> ExitCode: ) def _run_inner(self) -> ExitCode: - if self.options.builtin_or_auxiliary_goal: + if self.ng_invocation: + goals = self.ng_invocation.goals() + elif self.options.builtin_or_auxiliary_goal: return self._run_builtin_or_auxiliary_goal(self.options.builtin_or_auxiliary_goal) - - goals = tuple(self.options.goals) + else: + goals = tuple(self.options.goals) if not goals: return PANTS_SUCCEEDED_EXIT_CODE @@ -268,13 +287,8 @@ def _run_inner(self) -> ExitCode: return PANTS_FAILED_EXIT_CODE def run(self, start_time: float) -> ExitCode: - spec_parser = SpecsParser(working_dir=self.working_dir) - specs = [] - for spec_str in self.options.specs: - spec, is_ignore = spec_parser.parse_spec(spec_str) - specs.append(f"-{spec}" if is_ignore else str(spec)) - - self.run_tracker.start(run_start_time=start_time, specs=specs) + specs_strs = list(self.ng_invocation.specs()) if self.ng_invocation else self.options.specs + self.run_tracker.start(run_start_time=start_time, specs=specs_strs) global_options = self.options.for_global_scope() streaming_reporter = StreamingWorkunitHandler( diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 1b30330db59..cdf89bf4e81 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -4,10 +4,12 @@ import importlib import locale +import logging import os import sys import time import warnings +from datetime import datetime from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( @@ -19,6 +21,9 @@ from pants.engine.internals import native_engine from pants.util.strutil import softwrap +# pants: infer-dep(/src/python/pants/backend/**/register.py) +# pants: infer-dep(/src/python/pants/core/**/*) + class PantsLoader: """Initial entrypoint for pants. @@ -118,6 +123,20 @@ def run_default_entrypoint() -> None: @classmethod def main(cls) -> None: + # Set up logging. This is just temporary, as the PantsRunner will re-initialize logging to + # be emitted via Rust, but until that happens we want basic logging to do something useful. + # We set it up to be visually compatible with the Rust logging. + class HundredthsFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + dt = datetime.fromtimestamp(record.created) + formatted_time = dt.strftime(datefmt or "%H:%M:%S") + hundredths = str(dt.microsecond // 10000).zfill(2) + return f"{formatted_time}.{hundredths}" + + handler = logging.StreamHandler() + handler.setFormatter(HundredthsFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s")) + logging.basicConfig(level=logging.INFO, handlers=[handler]) + native_engine.initialize() cls.setup_warnings() cls.ensure_locale() diff --git a/src/python/pants/bin/pants_runner.py b/src/python/pants/bin/pants_runner.py index 4b956cb5395..5d7cd94b056 100644 --- a/src/python/pants/bin/pants_runner.py +++ b/src/python/pants/bin/pants_runner.py @@ -15,6 +15,7 @@ from pants.base.exception_sink import ExceptionSink from pants.base.exiter import ExitCode from pants.engine.env_vars import CompleteEnvironmentVars +from pants.engine.internals.native_engine import PyNgInvocation from pants.init.logging import initialize_stdio, stdio_destination from pants.init.util import init_workdir from pants.option.bootstrap_options import BootstrapOptions @@ -74,9 +75,27 @@ def scrub_pythonpath() -> None: def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() - options_bootstrapper = OptionsBootstrapper.create( - args=self.args, env=self.env, allow_pantsrc=True - ) + # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from + # config and env. We can't parse args until we know if we're ng or og. + options_bootstrapper = OptionsBootstrapper.create(args=[], env=self.env, allow_pantsrc=True) + pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng + + if pants_ng: + logger.info("PantsRunner running as pants_ng") + ng_invocation = PyNgInvocation.from_args(tuple(self.args[1:])) + # Allow the existing logic to read flags in global position (note, these can be + # prefixed with a scope so they are not necessarily just global options), so + # that the engine can configure itself. + global_flags = ng_invocation.global_flag_strings() + options_bootstrapper = OptionsBootstrapper.create( + args=[self.args[0], *global_flags], env=self.env, allow_pantsrc=True + ) + else: + ng_invocation = None + options_bootstrapper = OptionsBootstrapper.create( + args=self.args, env=self.env, allow_pantsrc=True + ) + with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() @@ -155,6 +174,7 @@ def run(self, start_time: float) -> ExitCode: env=CompleteEnvironmentVars(self.env), working_dir=os.getcwd(), options_bootstrapper=options_bootstrapper, + ng_invocation=ng_invocation, ) return runner.run(start_time) diff --git a/src/python/pants/build_graph/build_configuration.py b/src/python/pants/build_graph/build_configuration.py index 6d6227ed76b..19fa9ee007c 100644 --- a/src/python/pants/build_graph/build_configuration.py +++ b/src/python/pants/build_graph/build_configuration.py @@ -17,6 +17,7 @@ from pants.engine.rules import Rule, RuleIndex, collect_rules, rule from pants.engine.target import Target from pants.engine.unions import UnionRule +from pants.ng.goal import GoalSubsystemNg from pants.option.alias import CliOptions from pants.option.global_options import GlobalOptions from pants.option.scope import OptionsParsingSettings, ScopeInfo, normalize_scope @@ -143,6 +144,7 @@ class Builder: ) _allow_unknown_options: bool = False _remote_auth_plugin: Callable | None = None + _pants_ng: bool = False def registered_aliases(self) -> BuildFileAliases: """Return the registered aliases exposed in BUILD files. @@ -217,6 +219,15 @@ def register_subsystems( ) for subsystem in subsystems: + if ( + issubclass(subsystem, GoalSubsystem) + and issubclass(subsystem, GoalSubsystemNg) != self._pants_ng + ): + # Don't register goal subsystems that aren't pertinent to our og/ng state. + # This allows GoalSubsystemNgs to use the same scopes as og GoalSubsystems + # without collision. Note that the @rules that consume these subsystems + # can still be registered, but the subsystems cannot be configured. + continue self._subsystem_to_providers[subsystem].append(plugin_or_backend) def register_rules(self, plugin_or_backend: str, rules: Iterable[Rule | UnionRule]): diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index ff53c581fb7..26ffb69f70e 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -37,12 +37,13 @@ from pants.engine.internals.scheduler import Scheduler, SchedulerSession from pants.engine.internals.selectors import Params from pants.engine.internals.session import SessionValues -from pants.engine.rules import QueryRule, collect_rules, rule +from pants.engine.rules import QueryRule, Rule, collect_rules, rule from pants.engine.streaming_workunit_handler import rules as streaming_workunit_handler_rules from pants.engine.target import RegisteredTargetTypes from pants.engine.unions import UnionMembership, UnionRule from pants.init import specs_calculator from pants.init.bootstrap_scheduler import BootstrapStatus +from pants.ng.goal import GoalNg, GoalSubsystemNg from pants.option.bootstrap_options import ( DEFAULT_EXECUTION_OPTIONS, DynamicRemoteOptions, @@ -141,8 +142,13 @@ def run_goal_rules( env_name = determine_bootstrap_environment(self.scheduler_session) for goal in goals: - goal_product = self.goal_map[goal] - if not goal_product.subsystem_cls.activated(union_membership): + goal_product = self.goal_map.get(goal) + if goal_product is None: + # This must be an ng command, since the og goals have already + # been validated by the options parser. + raise UnknownCommand(goal) + pants_ng = issubclass(goal_product.subsystem_cls, GoalSubsystemNg) + if not pants_ng and not goal_product.subsystem_cls.activated(union_membership): raise GoalNotActivatedException(goal) # NB: Keep this in sync with the property `goal_param_types`. params = Params(specs, self.console, workspace, env_name) @@ -167,15 +173,21 @@ class GoalMappingError(Exception): """Raised when a goal cannot be mapped to an @rule.""" @staticmethod - def _make_goal_map_from_rules(rules) -> Mapping[str, type[Goal]]: + def _make_goal_map_from_rules(rules, pants_ng: bool) -> Mapping[str, type[Goal]]: goal_map: dict[str, type[Goal]] = {} for r in rules: output_type = getattr(r, "output_type", None) - if not output_type or not issubclass(output_type, Goal): + if ( + not output_type + or not issubclass(output_type, Goal) + or issubclass(output_type, GoalNg) != pants_ng + ): continue goal = r.output_type.name - deprecated_goal = r.output_type.subsystem_cls.deprecated_options_scope + deprecated_goal = ( + None if pants_ng else r.output_type.subsystem_cls.deprecated_options_scope + ) for goal_name in [goal, deprecated_goal] if deprecated_goal else [goal]: if goal_name in goal_map: raise EngineInitializer.GoalMappingError( @@ -220,6 +232,7 @@ def setup_graph( engine_visualize_to=bootstrap_options.engine_visualize_to, watch_filesystem=bootstrap_options.watch_filesystem, is_bootstrap=is_bootstrap, + pants_ng=bootstrap_options.pants_ng, ) @staticmethod @@ -240,6 +253,7 @@ def setup_graph_extended( engine_visualize_to: str | None = None, watch_filesystem: bool = True, is_bootstrap: bool = False, + pants_ng: bool = False, ) -> GraphScheduler: build_root_path = build_root or get_buildroot() @@ -309,7 +323,19 @@ async def current_executing_goals(session_values: SessionValues) -> CurrentExecu ) ) - goal_map = EngineInitializer._make_goal_map_from_rules(rules) + def is_pertinent_rule(rule: Rule | UnionRule, ng: bool) -> bool: + output_type = getattr(rule, "output_type", None) + if not output_type: # This is a UnionRule, so keep it. + return True + if issubclass(output_type, Goal) and issubclass(output_type, GoalNg) != ng: + return False + return True + + # Strip out all the goal_rules that don't pertain to our current ng/og state. This allows + # ng commands to use the same names as og goals, without collision. + rules = FrozenOrderedSet(rule for rule in rules if is_pertinent_rule(rule, pants_ng)) + + goal_map = EngineInitializer._make_goal_map_from_rules(rules, pants_ng) union_membership = UnionMembership.from_rules( ( @@ -370,6 +396,11 @@ def ensure_optional_absolute_path(v: str | None) -> str | None: return GraphScheduler(scheduler, goal_map) +class UnknownCommand(Exception): + def __init__(self, goal: str) -> None: + super().__init__(f"Unknown command `{goal.replace('.', ' ')}`") + + class GoalNotActivatedException(Exception): def __init__(self, goal_name: str) -> None: super().__init__( diff --git a/src/python/pants/init/extension_loader.py b/src/python/pants/init/extension_loader.py index 986c1676ffd..f24b9be1a03 100644 --- a/src/python/pants/init/extension_loader.py +++ b/src/python/pants/init/extension_loader.py @@ -34,18 +34,18 @@ class PluginLoadOrderError(PluginLoadingError): def load_backends_and_plugins( plugins: list[str], backends: list[str], - bc_builder: BuildConfiguration.Builder | None = None, + bc_builder: BuildConfiguration.Builder, ) -> BuildConfiguration: """Load named plugins and source backends. - :param plugins: v2 plugins to load. - :param backends: v2 backends to load. + :param plugins: plugins to load. + :param backends: backends to load. :param bc_builder: The BuildConfiguration (for adding aliases). """ - bc_builder = bc_builder or BuildConfiguration.Builder() load_build_configuration_from_source(bc_builder, backends) load_plugins(bc_builder, plugins) - register_builtin_goals(bc_builder) + if not bc_builder._pants_ng: + register_builtin_goals(bc_builder) return bc_builder.create() diff --git a/src/python/pants/init/options_initializer.py b/src/python/pants/init/options_initializer.py index 0c9830abd0a..b92ee353dd7 100644 --- a/src/python/pants/init/options_initializer.py +++ b/src/python/pants/init/options_initializer.py @@ -61,6 +61,7 @@ def _initialize_build_configuration( return load_backends_and_plugins( bootstrap_options.plugins, bootstrap_options.backend_packages, + BuildConfiguration.Builder(_pants_ng=bootstrap_options.pants_ng), ) diff --git a/src/python/pants/init/specs_calculator.py b/src/python/pants/init/specs_calculator.py index 4d1df82c830..53d5ae14fad 100644 --- a/src/python/pants/init/specs_calculator.py +++ b/src/python/pants/init/specs_calculator.py @@ -2,7 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -from typing import cast +from typing import Iterable, cast from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.base.specs import AddressLiteralSpec, FileLiteralSpec, RawSpecs, Specs @@ -31,6 +31,7 @@ class InvalidSpecConstraint(Exception): def calculate_specs( + specs_strs: Iterable[str], options_bootstrapper: OptionsBootstrapper, options: Options, session: SchedulerSession, @@ -40,7 +41,7 @@ def calculate_specs( global_options = options.for_global_scope() unmatched_cli_globs = global_options.unmatched_cli_globs specs = SpecsParser(working_dir=working_dir).parse_specs( - options.specs, + specs_strs, description_of_origin="CLI arguments", unmatched_glob_behavior=unmatched_cli_globs, ) diff --git a/src/python/pants/option/bootstrap_options.py b/src/python/pants/option/bootstrap_options.py index a6faebf3778..75894f90ec2 100644 --- a/src/python/pants/option/bootstrap_options.py +++ b/src/python/pants/option/bootstrap_options.py @@ -1781,6 +1781,19 @@ def file_downloads_max_attempts(self) -> int: ), ) + pants_ng = BoolOption( + default=False, + advanced=True, + help=softwrap( + """ + Enable the Pants "next generation" mode. + + Note that this is currently under development and is *not* yet useful to end users. + Do not set this unless you truly know what you're doing. + """ + ), + ) + @classmethod def validate_instance(cls, opts: OptionValueContainer): """Validates an instance of global options for cases that are not prohibited via