diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 8e42cca7e..2f7ebfe44 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -20,6 +20,7 @@ import shutil import click from .. import _yaml +from .._frontend.app import App from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope @@ -377,8 +378,6 @@ def cli(context, **kwargs): user preferences configuration file. """ - from .app import App - # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) @@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_): metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) +@click.option( + "--with", + "other_targets", + type=click.Path(readable=False), + multiple=True, + help="A additional target to stage into an element's sandbox environment", +) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", @@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_): @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( - app, + app: App, target, + other_targets, command, mount, isolate, @@ -748,13 +755,38 @@ def shell( otherwise bst may respond to them instead. e.g. \b - bst shell example.bst -- df -h + bst shell base.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. + Use the --with option to stage the artifacts of other elements + into the temporary sysroot to make them available to run e.g. + + \b + bst shell --with base.bst example.bst -- cat example.txt + If no COMMAND is specified, the default is to attempt to run an interactive shell. + + # Examples: + + \b + # Attempt to run an interactive shell with example.bst + bst shell example.bst + # Attempt to run an df -h with example.bst + bst shell example.bst -- df h + # In a workspace directory, attempt to shell into the workspace element + bst shell + # Attempt to run cat from base.bst to read example.txt from example.bst + bst shell --with base.bst example.bst -- cat example.txt + # Attempt to run an interactive shell with the sources and all dependencies of example.bst + bst shell --build example.bst + + For all examples on this page: + - example.bst is a simple import element with no dependencies + that imports a file called example.txt + - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). """ # Buildtree can only be used with build shells @@ -764,6 +796,7 @@ def shell( scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): + assert app.stream, "Must have Stream initialised" if not target: target = app.stream.get_default_target() if not target: @@ -772,19 +805,35 @@ def shell( mounts = [_HostMount(path, host_path) for host_path, path in mount] try: - exitcode = app.stream.shell( - target, - scope, - app.shell_prompt, - mounts=mounts, - isolate=isolate, - command=command, - usebuildtree=cli_buildtree, - artifact_remotes=artifact_remotes, - source_remotes=source_remotes, - ignore_project_artifact_remotes=ignore_project_artifact_remotes, - ignore_project_source_remotes=ignore_project_source_remotes, - ) + if other_targets: + exitcode = app.stream.shell_with( + target, + other_targets, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) + else: + exitcode = app.stream.shell( + target, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index af98504cc..4bdd53905 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -21,7 +21,7 @@ from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element -from ..node import Node +from ..node import Node, MappingNode from .._profile import Topics, PROFILER from .._includes import Includes from .._utils import valid_chars_name @@ -1014,7 +1014,9 @@ def _shallow_load_path(self, path, provenance_node): # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, provenance_node, *, load_subprojects=True): + def _parse_name( + self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True + ) -> tuple[str | None, str, "Loader"]: # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 91971073f..3d98be0fc 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -16,6 +16,7 @@ # Jürg Billeter # Tristan Maat + import itertools import os import sys @@ -27,7 +28,11 @@ from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable +from ruamel.yaml import CommentedMap + +from ._context import Context +from .node import MappingNode from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError @@ -44,7 +49,7 @@ ) from .element import Element from ._profile import Topics, PROFILER -from ._project import ProjectRefStorage +from ._project import ProjectRefStorage, Project from ._remotespec import RemoteSpec from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount @@ -79,11 +84,11 @@ def __init__( # # Private members # - self._context = context + self._context: Context = context self._artifacts = None self._elementsourcescache = None self._sourcecache = None - self._project = None + self._project: Optional[Project] = None self._state = State(session_start) # Owned by Stream, used by Core to set state self._notification_queue = deque() @@ -163,7 +168,7 @@ def load_selection( ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, - ): + ) -> list[Element]: with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, @@ -235,6 +240,69 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source task.add_current_progress() + # shell_with() + # + # Run a shell with other targets. + # + # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies. + # + # Note: Method will build the temporary target, before entering into it's shell. + # + # Args: + # target (str): The name of the element to run the shell for + # other_targets: (Iterable[str]): The name of the other elements to run the shell with. + # scope: _Scope: Either BUILD or RUN + # *args, **kwargs: Passed to shell() untouched. + # + # Returns: + # (int): The exit code of the launched shell + # + def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs): + + assert self._project, "Must have a project" + assert self._project.loader, "Project must have loader" + + target_junction, target_name, target_loader = self._project.loader._parse_name( + target, MappingNode.from_dict({}) + ) + + target_path = os.path.join(target_loader._basedir, target_name) + target_node: CommentedMap = _yaml.roundtrip_load(target_path) + + if scope == _Scope.RUN: + r_depends = target_node.get("runtime-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["runtime-depends"] = r_depends + elif scope == _Scope.BUILD: + r_depends = target_node.get("build-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["build-depends"] = r_depends + else: + raise StreamError( + "Only BUILD and RUN scopes are supported", + detail="Use the --build and --use-buildtree options to shell into a build tree", + reason="only-build-run-supported", + ) + + with tempfile.NamedTemporaryFile( + dir=target_loader._basedir, delete_on_close=False, prefix=f"{target_name}_temp", suffix=".bst" + ) as temp_target_file: + _yaml.roundtrip_dump(target_node, temp_target_file) + temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. + + new_target = os.path.relpath(temp_target_file.name, target_loader._basedir) + if target_junction: + new_target = f"{target_junction}:{new_target}" + + self.build([new_target]) + return self.shell(new_target, scope, *args, **kwargs) + # shell() # # Run a shell @@ -243,6 +311,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell + # other_targets (Iterable[str]): The name of other elements to stage in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds @@ -1043,6 +1112,7 @@ def workspace_open( self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) if not custom_dir: + assert self._context.workspacedir, "Must have workspace dir" directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) if directory[-4:] == ".bst": directory = directory[:-4] @@ -2114,6 +2184,8 @@ def _expand_and_classify_targets( # project directory and element path prefix, to produce only element names. # all_elements = [] + assert self._project, "Must have a project" + assert self._project.element_path, "Must have a project" element_path_length = len(self._project.element_path) + 1 for dirpath, _, filenames in os.walk(self._project.element_path): for filename in filenames: @@ -2133,6 +2205,7 @@ def _expand_and_classify_targets( # Glob the artifact names and add the results to the set # + assert self._artifacts, "Must have artifacts" for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: diff --git a/src/buildstream/_yaml.pyi b/src/buildstream/_yaml.pyi index 224abc51a..b301cab68 100644 --- a/src/buildstream/_yaml.pyi +++ b/src/buildstream/_yaml.pyi @@ -12,7 +12,10 @@ # limitations under the License. # from typing import Optional +from ruamel.yaml import CommentedMap from .node import MappingNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... +def roundtrip_load(filename: str, *, allow_missing: bool = False) -> CommentedMap: ... +def roundtrip_dump(contents, file): ... diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 209b9e9a9..62d74eb1d 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,6 +62,9 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations + import os import re import stat @@ -74,6 +77,7 @@ from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence + from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml @@ -2058,6 +2062,7 @@ def _push(self): # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source + # other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox # # Returns: Exit code def _shell( diff --git a/src/buildstream/types.pyi b/src/buildstream/types.pyi index 8a75b9f13..db9d59d06 100644 --- a/src/buildstream/types.pyi +++ b/src/buildstream/types.pyi @@ -116,10 +116,10 @@ class OverlapAction(Enum): IGNORE: str class _Scope(Enum): - ALL: int - BUILD: int - RUN: int - NONE: int + ALL = 1 + BUILD = 2 + RUN = 3 + NONE = 4 class _KeyStrength(Enum): STRONG: int diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..4d0b4ce6b 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,12 +16,15 @@ # pylint: disable=redefined-outer-name import os +from typing import Dict, List, Tuple import uuid + import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,11 +49,25 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` -# -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +# other_elements (list(str)): Other elements to stage in the sandbox +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: None | Dict = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + other_elements: List[str] | None = None +): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 + if other_elements is not None: + for other_element in other_elements: + result = cli.run_project_config(project=project, project_config=config, args=["build", other_element]) + assert result.exit_code == 0 args = ["shell"] if isolate: @@ -58,6 +75,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] + if other_elements is not None: + for other_element in other_elements: + args += ["--with", other_element] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) @@ -86,6 +106,28 @@ def test_executable(cli, datafiles): assert result.output == "Horseys!\n" +# Test staging and running additional targets in the shell of the main target for debugging. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_with_other_targets(cli, datafiles): + project = str(datafiles) + + # Show we can't cat in a shell for manual/import-file.bst + result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst") + assert ( + result.exit_code == -1 + ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies" + + # Show we can now cat with base.bst in a shell for manual/import-file.bst + result = execute_shell( + cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"] + ) + assert ( + result.exit_code == 0 + ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat" + assert result.output == "This is a test\n" + + # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR)