[pants_ng] Scaffolding for a pants_ng mode.

src/python/pants/bin/daemon_pants_runner.py

@@ -11,7 +11,7 @@
 from pants.base.exiter import PANTS_FAILED_EXIT_CODE, ExitCode
 from pants.bin.local_pants_runner import LocalPantsRunner
 from pants.engine.env_vars import CompleteEnvironmentVars
-from pants.engine.internals.native_engine import PySessionCancellationLatch
+from pants.engine.internals.native_engine import PyNgInvocation, PySessionCancellationLatch
 from pants.init.logging import stdio_destination
 from pants.option.options_bootstrapper import OptionsBootstrapper
 from pants.pantsd.pants_daemon_core import PantsDaemonCore
@@ -107,6 +107,10 @@ def single_daemonized_run(
         method should not need any special handling for the fact that it's running in a proxied
         environment.
         """
+        # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from
+        # config and env. We can't parse args until we know if we're ng or og.
+        options_bootstrapper = OptionsBootstrapper.create(args=[], env=env, allow_pantsrc=True)
+        pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng
 
         try:
             logger.debug("Connected to pantsd")
@@ -123,9 +127,21 @@ def single_daemonized_run(
                 )
             start_time = float(env_start_time) if env_start_time else time.time()
 
-            options_bootstrapper = OptionsBootstrapper.create(
-                args=args, env=env, allow_pantsrc=True
-            )
+            if pants_ng:
+                logger.info("DaemonPantsRunner running as pants_ng")
+                ng_invocation = PyNgInvocation.from_args(args[1:])
+                # Allow the existing logic to read flags in global position (note, these can be
+                # prefixed with a scope so they are not necessarily just global options), so
+                # that the engine can configure itself.
+                global_flags = ng_invocation.global_flag_strings()
+                options_bootstrapper = OptionsBootstrapper.create(
+                    args=[args[0], *global_flags], env=env, allow_pantsrc=True
+                )
+            else:
+                ng_invocation = None
+                options_bootstrapper = OptionsBootstrapper.create(
+                    args=args, env=env, allow_pantsrc=True
+                )
 
             # Run using the pre-warmed Session.
             complete_env = CompleteEnvironmentVars(env)
@@ -137,6 +153,7 @@ def single_daemonized_run(
                 scheduler=scheduler,
                 options_initializer=options_initializer,
                 cancellation_latch=cancellation_latch,
+                ng_invocation=ng_invocation,
             )
             return runner.run(start_time)
         except Exception as e:

src/python/pants/bin/local_pants_runner.py

@@ -10,13 +10,17 @@
 
 from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode
 from pants.base.specs import Specs
-from pants.base.specs_parser import SpecsParser
 from pants.build_graph.build_configuration import BuildConfiguration
 from pants.core.environments.rules import determine_bootstrap_environment
 from pants.engine.env_vars import CompleteEnvironmentVars
 from pants.engine.goal import CurrentExecutingGoals
 from pants.engine.internals import native_engine
-from pants.engine.internals.native_engine import PyExecutor, PySessionCancellationLatch
+from pants.engine.internals.native_engine import (
+    PyExecutor,
+    PyNgInvocation,
+    PyNgOptions,
+    PySessionCancellationLatch,
+)
 from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.internals.selectors import Params
 from pants.engine.internals.session import SessionValues
@@ -63,6 +67,7 @@ class LocalPantsRunner:
     union_membership: UnionMembership
     is_pantsd_run: bool
     working_dir: str
+    ng_invocation: PyNgInvocation | None
 
     @classmethod
     def create(
@@ -73,6 +78,7 @@ def create(
         options_initializer: OptionsInitializer | None = None,
         scheduler: GraphScheduler | None = None,
         cancellation_latch: PySessionCancellationLatch | None = None,
+        ng_invocation: PyNgInvocation | None = None,
     ) -> LocalPantsRunner:
         """Creates a new LocalPantsRunner instance by parsing options.
 
@@ -126,8 +132,18 @@ def create(
             )
         with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
             global_options = options.for_global_scope()
+            session_values_dict = {
+                OptionsBootstrapper: options_bootstrapper,
+                CompleteEnvironmentVars: env,
+                CurrentExecutingGoals: CurrentExecutingGoals(),
+            }
+            if ng_invocation is not None:
+                session_values_dict[PyNgInvocation] = ng_invocation
+                session_values_dict[PyNgOptions] = PyNgOptions(
+                    ng_invocation, dict(env.items()), include_derivation=False
+                )
         graph_session = scheduler.new_session(
-            run_tracker.run_id,
+            build_id=run_tracker.run_id,
             dynamic_ui=global_options.dynamic_ui,
             ui_use_prodash=global_options.dynamic_ui_renderer
             == DynamicUIRenderer.experimental_prodash,
@@ -140,17 +156,17 @@ def create(
                     for level in global_options.log_levels_by_target.values()
                 ),
             ),
-            session_values=SessionValues(
-                {
-                    OptionsBootstrapper: options_bootstrapper,
-                    CompleteEnvironmentVars: env,
-                    CurrentExecutingGoals: CurrentExecutingGoals(),
-                }
-            ),
+            session_values=SessionValues(session_values_dict),
             cancellation_latch=cancellation_latch,
         )
 
+        if ng_invocation:
+            specs_strs = ng_invocation.specs()
+        else:
+            specs_strs = tuple(options.specs)
+
         specs = calculate_specs(
+            specs_strs=specs_strs,
             options_bootstrapper=options_bootstrapper,
             options=options,
             session=graph_session.scheduler_session,
@@ -169,6 +185,7 @@ def create(
             union_membership=union_membership,
             is_pantsd_run=is_pantsd_run,
             working_dir=working_dir,
+            ng_invocation=ng_invocation,
         )
 
     def _perform_run(self, goals: tuple[str, ...]) -> ExitCode:
@@ -251,10 +268,12 @@ def _run_auxiliary_goal(context: AuxiliaryGoalContext, goal: Any) -> ExitCode:
             )
 
     def _run_inner(self) -> ExitCode:
-        if self.options.builtin_or_auxiliary_goal:
+        if self.ng_invocation:
+            goals = self.ng_invocation.goals()
+        elif self.options.builtin_or_auxiliary_goal:
             return self._run_builtin_or_auxiliary_goal(self.options.builtin_or_auxiliary_goal)
-
-        goals = tuple(self.options.goals)
+        else:
+            goals = tuple(self.options.goals)
         if not goals:
             return PANTS_SUCCEEDED_EXIT_CODE
 
@@ -268,13 +287,8 @@ def _run_inner(self) -> ExitCode:
             return PANTS_FAILED_EXIT_CODE
 
     def run(self, start_time: float) -> ExitCode:
-        spec_parser = SpecsParser(working_dir=self.working_dir)
-        specs = []
-        for spec_str in self.options.specs:
-            spec, is_ignore = spec_parser.parse_spec(spec_str)
-            specs.append(f"-{spec}" if is_ignore else str(spec))
-
-        self.run_tracker.start(run_start_time=start_time, specs=specs)
+        specs_strs = list(self.ng_invocation.specs()) if self.ng_invocation else self.options.specs
+        self.run_tracker.start(run_start_time=start_time, specs=specs_strs)
         global_options = self.options.for_global_scope()
 
         streaming_reporter = StreamingWorkunitHandler(

src/python/pants/bin/pants_loader.py

@@ -4,10 +4,12 @@
 
 import importlib
 import locale
+import logging
 import os
 import sys
 import time
 import warnings
+from datetime import datetime
 
 from pants.base.exiter import PANTS_FAILED_EXIT_CODE
 from pants.bin.pants_env_vars import (
@@ -19,6 +21,9 @@
 from pants.engine.internals import native_engine
 from pants.util.strutil import softwrap
 
+# pants: infer-dep(/src/python/pants/backend/**/register.py)
+# pants: infer-dep(/src/python/pants/core/**/*)
+
 
 class PantsLoader:
     """Initial entrypoint for pants.
@@ -118,6 +123,20 @@ def run_default_entrypoint() -> None:
 
     @classmethod
     def main(cls) -> None:
+        # Set up logging. This is just temporary, as the PantsRunner will re-initialize logging to
+        # be emitted via Rust, but until that happens we want basic logging to do something useful.
+        # We set it up to be visually compatible with the Rust logging.
+        class HundredthsFormatter(logging.Formatter):
+            def formatTime(self, record, datefmt=None):
+                dt = datetime.fromtimestamp(record.created)
+                formatted_time = dt.strftime(datefmt or "%H:%M:%S")
+                hundredths = str(dt.microsecond // 10000).zfill(2)
+                return f"{formatted_time}.{hundredths}"
+
+        handler = logging.StreamHandler()
+        handler.setFormatter(HundredthsFormatter(fmt="%(asctime)s [%(levelname)s] %(message)s"))
+        logging.basicConfig(level=logging.INFO, handlers=[handler])
+
         native_engine.initialize()
         cls.setup_warnings()
         cls.ensure_locale()

src/python/pants/bin/pants_runner.py

@@ -15,6 +15,7 @@
 from pants.base.exception_sink import ExceptionSink
 from pants.base.exiter import ExitCode
 from pants.engine.env_vars import CompleteEnvironmentVars
+from pants.engine.internals.native_engine import PyNgInvocation
 from pants.init.logging import initialize_stdio, stdio_destination
 from pants.init.util import init_workdir
 from pants.option.bootstrap_options import BootstrapOptions
@@ -74,9 +75,27 @@ def scrub_pythonpath() -> None:
     def run(self, start_time: float) -> ExitCode:
         self.scrub_pythonpath()
 
-        options_bootstrapper = OptionsBootstrapper.create(
-            args=self.args, env=self.env, allow_pantsrc=True
-        )
+        # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from
+        # config and env. We can't parse args until we know if we're ng or og.
+        options_bootstrapper = OptionsBootstrapper.create(args=[], env=self.env, allow_pantsrc=True)
+        pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng
+
+        if pants_ng:
+            logger.info("PantsRunner running as pants_ng")
+            ng_invocation = PyNgInvocation.from_args(tuple(self.args[1:]))
+            # Allow the existing logic to read flags in global position (note, these can be
+            # prefixed with a scope so they are not necessarily just global options), so
+            # that the engine can configure itself.
+            global_flags = ng_invocation.global_flag_strings()
+            options_bootstrapper = OptionsBootstrapper.create(
+                args=[self.args[0], *global_flags], env=self.env, allow_pantsrc=True
+            )
+        else:
+            ng_invocation = None
+            options_bootstrapper = OptionsBootstrapper.create(
+                args=self.args, env=self.env, allow_pantsrc=True
+            )
+
         with warnings.catch_warnings(record=True):
             bootstrap_options = options_bootstrapper.bootstrap_options
             global_bootstrap_options = bootstrap_options.for_global_scope()
@@ -155,6 +174,7 @@ def run(self, start_time: float) -> ExitCode:
                 env=CompleteEnvironmentVars(self.env),
                 working_dir=os.getcwd(),
                 options_bootstrapper=options_bootstrapper,
+                ng_invocation=ng_invocation,
             )
             return runner.run(start_time)
 

src/python/pants/build_graph/build_configuration.py

@@ -17,6 +17,7 @@
 from pants.engine.rules import Rule, RuleIndex, collect_rules, rule
 from pants.engine.target import Target
 from pants.engine.unions import UnionRule
+from pants.ng.goal import GoalSubsystemNg
 from pants.option.alias import CliOptions
 from pants.option.global_options import GlobalOptions
 from pants.option.scope import OptionsParsingSettings, ScopeInfo, normalize_scope
@@ -143,6 +144,7 @@ class Builder:
         )
         _allow_unknown_options: bool = False
         _remote_auth_plugin: Callable | None = None
+        _pants_ng: bool = False
 
         def registered_aliases(self) -> BuildFileAliases:
             """Return the registered aliases exposed in BUILD files.
@@ -217,6 +219,15 @@ def register_subsystems(
                 )
 
             for subsystem in subsystems:
+                if (
+                    issubclass(subsystem, GoalSubsystem)
+                    and issubclass(subsystem, GoalSubsystemNg) != self._pants_ng
+                ):
+                    # Don't register goal subsystems that aren't pertinent to our og/ng state.
+                    # This allows GoalSubsystemNgs to use the same scopes as og GoalSubsystems
+                    # without collision. Note that the @rules that consume these subsystems
+                    # can still be registered, but the subsystems cannot be configured.
+                    continue
                 self._subsystem_to_providers[subsystem].append(plugin_or_backend)
 
         def register_rules(self, plugin_or_backend: str, rules: Iterable[Rule | UnionRule]):