Shell API and CLI: Add option for staging additional runtime targets

src/buildstream/_frontend/cli.py

@@ -20,6 +20,7 @@
 import shutil
 import click
 from .. import _yaml
+from .._frontend.app import App
 from .._exceptions import BstError, LoadError, AppError, RemoteError
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
@@ -377,8 +378,6 @@ def cli(context, **kwargs):
     user preferences configuration file.
     """
 
-    from .app import App
-
     # Create the App, giving it the main arguments
     context.obj = App.create(dict(kwargs))
     context.call_on_close(context.obj.cleanup)
@@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_):
     metavar="HOSTPATH PATH",
     help="Mount a file or directory into the sandbox",
 )
+@click.option(
+    "--with",
+    "other_targets",
+    type=click.Path(readable=False),
+    multiple=True,
+    help="A additional target to stage into an element's sandbox environment",
+)
 @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
 @click.option(
     "--use-buildtree",
@@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_):
 @click.argument("command", type=click.STRING, nargs=-1)
 @click.pass_obj
 def shell(
-    app,
+    app: App,
     target,
+    other_targets,
     command,
     mount,
     isolate,
@@ -748,13 +755,38 @@ def shell(
     otherwise bst may respond to them instead. e.g.
 
     \b
-        bst shell example.bst -- df -h
+        bst shell base.bst -- df -h
 
     Use the --build option to create a temporary sysroot for
     building the element instead.
 
+    Use the --with option to stage the artifacts of other elements
+    into the temporary sysroot to make them available to run e.g.
+
+    \b
+        bst shell --with base.bst example.bst -- cat example.txt
+
     If no COMMAND is specified, the default is to attempt
     to run an interactive shell.
+
+    # Examples:
+
+    \b
+        # Attempt to run an interactive shell with example.bst
+        bst shell example.bst
+        # Attempt to run an df -h with example.bst
+        bst shell example.bst -- df h
+        # In a workspace directory, attempt to shell into the workspace element
+        bst shell
+        # Attempt to run cat from base.bst to read example.txt from example.bst
+        bst shell --with base.bst example.bst -- cat example.txt
+        # Attempt to run an interactive shell with the sources and all dependencies of example.bst
+        bst shell --build example.bst
+
+    For all examples on this page:
+    - example.bst is a simple import element with no dependencies
+       that imports a file called example.txt
+    - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).
     """
 
     # Buildtree can only be used with build shells
@@ -764,6 +796,7 @@ def shell(
     scope = _Scope.BUILD if build_ else _Scope.RUN
 
     with app.initialized():
+        assert app.stream, "Must have Stream initialised"
         if not target:
             target = app.stream.get_default_target()
             if not target:
@@ -772,19 +805,35 @@ def shell(
         mounts = [_HostMount(path, host_path) for host_path, path in mount]
 
         try:
-            exitcode = app.stream.shell(
-                target,
-                scope,
-                app.shell_prompt,
-                mounts=mounts,
-                isolate=isolate,
-                command=command,
-                usebuildtree=cli_buildtree,
-                artifact_remotes=artifact_remotes,
-                source_remotes=source_remotes,
-                ignore_project_artifact_remotes=ignore_project_artifact_remotes,
-                ignore_project_source_remotes=ignore_project_source_remotes,
-            )
+            if other_targets:
+                exitcode = app.stream.shell_with(
+                    target,
+                    other_targets,
+                    scope,
+                    app.shell_prompt,
+                    mounts=mounts,
+                    isolate=isolate,
+                    command=command,
+                    usebuildtree=cli_buildtree,
+                    artifact_remotes=artifact_remotes,
+                    source_remotes=source_remotes,
+                    ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                    ignore_project_source_remotes=ignore_project_source_remotes,
+                )
+            else:
+                exitcode = app.stream.shell(
+                    target,
+                    scope,
+                    app.shell_prompt,
+                    mounts=mounts,
+                    isolate=isolate,
+                    command=command,
+                    usebuildtree=cli_buildtree,
+                    artifact_remotes=artifact_remotes,
+                    source_remotes=source_remotes,
+                    ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                    ignore_project_source_remotes=ignore_project_source_remotes,
+                )
         except BstError as e:
             raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e
 

src/buildstream/_loader/loader.py

@@ -21,7 +21,7 @@
 from ..exceptions import LoadErrorReason
 from .. import _yaml
 from ..element import Element
-from ..node import Node
+from ..node import Node, MappingNode
 from .._profile import Topics, PROFILER
 from .._includes import Includes
 from .._utils import valid_chars_name
@@ -1014,7 +1014,9 @@ def _shallow_load_path(self, path, provenance_node):
     #            - (str): name of the element
     #            - (Loader): loader for sub-project
     #
-    def _parse_name(self, name, provenance_node, *, load_subprojects=True):
+    def _parse_name(
+        self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True
+    ) -> tuple[str | None, str, "Loader"]:
         # We allow to split only once since deep junctions names are forbidden.
         # Users who want to refer to elements in sub-sub-projects are required
         # to create junctions on the top level project.

src/buildstream/_stream.py

@@ -16,6 +16,7 @@
 #        Jürg Billeter <juerg.billeter@codethink.co.uk>
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 
+
 import itertools
 import os
 import sys
@@ -27,7 +28,11 @@
 from contextlib import contextmanager, suppress
 from collections import deque
 from typing import List, Tuple, Optional, Iterable, Callable
+from ruamel.yaml import CommentedMap
+
 
+from ._context import Context
+from .node import MappingNode
 from ._artifactelement import verify_artifact_ref, ArtifactElement
 from ._artifactproject import ArtifactProject
 from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
@@ -44,7 +49,7 @@
 )
 from .element import Element
 from ._profile import Topics, PROFILER
-from ._project import ProjectRefStorage
+from ._project import ProjectRefStorage, Project
 from ._remotespec import RemoteSpec
 from ._state import State
 from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount
@@ -79,11 +84,11 @@ def __init__(
         #
         # Private members
         #
-        self._context = context
+        self._context: Context = context
         self._artifacts = None
         self._elementsourcescache = None
         self._sourcecache = None
-        self._project = None
+        self._project: Optional[Project] = None
         self._state = State(session_start)  # Owned by Stream, used by Core to set state
         self._notification_queue = deque()
 
@@ -163,7 +168,7 @@ def load_selection(
         ignore_project_artifact_remotes: bool = False,
         ignore_project_source_remotes: bool = False,
         need_state: bool = True,
-    ):
+    ) -> list[Element]:
         with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)):
             target_objects = self._load(
                 targets,
@@ -235,6 +240,69 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
 
                     task.add_current_progress()
 
+    # shell_with()
+    #
+    # Run a shell with other targets.
+    #
+    # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies.
+    #
+    # Note: Method will build the temporary target, before entering into it's shell.
+    #
+    # Args:
+    #    target (str): The name of the element to run the shell for
+    #    other_targets: (Iterable[str]): The name of the other elements to run the shell with.
+    #    scope: _Scope: Either BUILD or RUN
+    #    *args, **kwargs:  Passed to shell() untouched.
+    #
+    # Returns:
+    #    (int): The exit code of the launched shell
+    #
+    def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs):
+
+        assert self._project, "Must have a project"
+        assert self._project.loader, "Project must have loader"
+
+        target_junction, target_name, target_loader = self._project.loader._parse_name(
+            target, MappingNode.from_dict({})
+        )
+
+        target_path = os.path.join(target_loader._basedir, target_name)
+        target_node: CommentedMap = _yaml.roundtrip_load(target_path)
+
+        if scope == _Scope.RUN:
+            r_depends = target_node.get("runtime-depends", [])
+
+            for other_target in other_targets:
+                r_depends.append(other_target)
+
+            target_node["runtime-depends"] = r_depends
+        elif scope == _Scope.BUILD:
+            r_depends = target_node.get("build-depends", [])
+
+            for other_target in other_targets:
+                r_depends.append(other_target)
+
+            target_node["build-depends"] = r_depends
+        else:
+            raise StreamError(
+                "Only BUILD and RUN scopes are supported",
+                detail="Use the --build and --use-buildtree options to shell into a build tree",
+                reason="only-build-run-supported",
+            )
+
+        with tempfile.NamedTemporaryFile(
+            dir=target_loader._basedir, delete_on_close=False, prefix=f"{target_name}_temp", suffix=".bst"
+        ) as temp_target_file:
+            _yaml.roundtrip_dump(target_node, temp_target_file)
+            temp_target_file.close()  # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager.
+
+            new_target = os.path.relpath(temp_target_file.name, target_loader._basedir)
+            if target_junction:
+                new_target = f"{target_junction}:{new_target}"
+
+            self.build([new_target])
+            return self.shell(new_target, scope, *args, **kwargs)
+
     # shell()
     #
     # Run a shell
@@ -243,6 +311,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     #    target: The name of the element to run the shell for
     #    scope: The scope for the shell, only BUILD or RUN are valid (_Scope)
     #    prompt: A function to return the prompt to display in the shell
+    #    other_targets (Iterable[str]): The name of other elements to stage in the shell
     #    unique_id: (str): A unique_id to use to lookup an Element instance
     #    mounts: Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
@@ -1043,6 +1112,7 @@ def workspace_open(
                 self.workspace_close(target._get_full_name(), remove_dir=not no_checkout)
 
             if not custom_dir:
+                assert self._context.workspacedir, "Must have workspace dir"
                 directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name))
                 if directory[-4:] == ".bst":
                     directory = directory[:-4]
@@ -2114,6 +2184,8 @@ def _expand_and_classify_targets(
             # project directory and element path prefix, to produce only element names.
             #
             all_elements = []
+            assert self._project, "Must have a project"
+            assert self._project.element_path, "Must have a project"
             element_path_length = len(self._project.element_path) + 1
             for dirpath, _, filenames in os.walk(self._project.element_path):
                 for filename in filenames:
@@ -2133,6 +2205,7 @@ def _expand_and_classify_targets(
 
         # Glob the artifact names and add the results to the set
         #
+        assert self._artifacts, "Must have artifacts"
         for glob in artifact_globs:
             glob_results = self._artifacts.list_artifacts(glob=glob)
             for artifact_name in glob_results:

src/buildstream/_yaml.pyi

@@ -12,7 +12,10 @@
 #  limitations under the License.
 #
 from typing import Optional
+from ruamel.yaml import CommentedMap
 
 from .node import MappingNode
 
 def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ...
+def roundtrip_load(filename: str, *, allow_missing: bool = False) -> CommentedMap: ...
+def roundtrip_dump(contents, file): ...

src/buildstream/element.py

@@ -62,6 +62,9 @@
 ---------------
 """
 
+# For 3.7+ support, not necessary and deprecated in 3.14+
+from __future__ import annotations
+
 import os
 import re
 import stat
@@ -74,6 +77,7 @@
 from threading import Lock
 from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence
 
+
 from pyroaring import BitMap  # pylint: disable=no-name-in-module
 
 from . import _yaml
@@ -2058,6 +2062,7 @@ def _push(self):
     #    prompt (str): A suitable prompt string for PS1
     #    command (list): An argv to launch in the sandbox
     #    usebuildtree (bool): Use the buildtree as its source
+    #    other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox
     #
     # Returns: Exit code
     def _shell(

src/buildstream/types.pyi

@@ -116,10 +116,10 @@ class OverlapAction(Enum):
     IGNORE: str
 
 class _Scope(Enum):
-    ALL: int
-    BUILD: int
-    RUN: int
-    NONE: int
+    ALL = 1
+    BUILD = 2
+    RUN = 3
+    NONE = 4
 
 class _KeyStrength(Enum):
     STRONG: int