Export: trimming enhancements

docs/commands.html

@@ -801,6 +801,11 @@ <h2 id="query" class="title-2">
         <code class="code">whatoutputs</code>: Prints out target(s) responsible for outputting provided file(s)
       </span>
     </li>
+    <li>
+      <span>
+        <code class="code">metadata</code>: Prints out a structured, tree-like visualization of parsed build statement metadata (required subincludes, files, and generated targets) of a package.
+      </span>
+    </li>
   </ul>
 
   <p>
@@ -1032,15 +1037,6 @@ <h3 class="mt1 f6 lh-title">
         </h3>
         <p>Disables trimming unnecessary targets from exported packages. Normally targets in exported packages that
           aren't dependencies of the originally exported targets are removed.</p>
-        <p>
-          This trimming syntax based, so doesn't always work depending on how the build definition is authored. Passing
-          this flag will disable this feature, avoiding cases where these rules will be erroneously trimmed.
-        </p>
-        <p>
-          To make sure a rule works without this flag, the rule must follow the naming convention, whereby children of
-          <code class="code">:name</code> follow the format <code class="code">:_name#{some-tag}</code>. This is the
-          format <code class="code">tag(name, tag)</code> would produce.
-        </p>
       </div>
     </li>
   </ul>

src/cli/logging.go

@@ -13,7 +13,7 @@ import (
 	"sync"
 
 	cli "github.com/peterebden/go-cli-init/v5/logging"
-	"github.com/peterebden/go-deferred-regex"
+	deferredregex "github.com/peterebden/go-deferred-regex"
 	"golang.org/x/term"
 	"gopkg.in/op/go-logging.v1"
 
@@ -184,9 +184,6 @@ func (backend *LogBackend) calcOutput() []string {
 			ret = append(ret, new...)
 		}
 	}
-	if len(ret) > 0 {
-		ret = append(ret, "Messages:")
-	}
 	return reverse(ret)
 }
 
@@ -248,6 +245,16 @@ func (backend *LogBackend) Output() []string {
 	return backend.output[:]
 }
 
+// FlushOutput returns the current set of preformatted log messages and clears them.
+func (backend *LogBackend) FlushOutput() []string {
+	backend.mutex.Lock()
+	defer backend.mutex.Unlock()
+	ret := backend.output[:]
+	backend.output = nil
+	backend.logMessages.Init()
+	return ret
+}
+
 // Wraps a string across multiple lines. Returned slice is reversed.
 func (backend *LogBackend) lineWrap(msg string) []string {
 	lines := strings.Split(msg, "\n")

src/core/build_label.go

@@ -473,7 +473,8 @@ func subrepoLabel(subrepoName, arch string) BuildLabel {
 	return BuildLabel{Name: subrepoName, Subrepo: arch}
 }
 
-func hashBuildLabel(l BuildLabel) uint64 {
+// HashBuildLabel calculates an hash of Build Label, suitable to use for map indexing.
+func HashBuildLabel(l BuildLabel) uint64 {
 	return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name)
 }
 
@@ -622,3 +623,15 @@ func (slice BuildLabels) String() string {
 	}
 	return strings.Join(s, ", ")
 }
+
+// labelSet defines a set of labels implemented using a map.
+type labelSet map[BuildLabel]struct{}
+
+func (ls labelSet) Add(l BuildLabel) {
+	ls[l] = struct{}{}
+}
+
+func (ls labelSet) Contains(l BuildLabel) bool {
+	_, ok := ls[l]
+	return ok
+}

src/core/build_target.go

@@ -1473,7 +1473,7 @@ func (target *BuildTarget) AllTestTools() []BuildInput {
 	if target.Test.namedTools == nil {
 		return target.Test.tools
 	}
-	return target.allBuildInputs(target.Test.tools, target.Test.namedTools)
+	return combinedBuildInputs(target.Test.tools, target.Test.namedTools)
 }
 
 // NamedTestTools returns all named test tools
@@ -1489,7 +1489,7 @@ func (target *BuildTarget) AllDebugTools() []BuildInput {
 	if target.Debug.namedTools == nil {
 		return target.Debug.tools
 	}
-	return target.allBuildInputs(target.Debug.tools, target.Debug.namedTools)
+	return combinedBuildInputs(target.Debug.tools, target.Debug.namedTools)
 }
 
 // AddDatum adds a new item of data to the target.
@@ -1644,21 +1644,41 @@ func (target *BuildTarget) getCommand(state *BuildState, commands map[string]str
 	return highestCommand
 }
 
+// AllBuildInputs returns all the inputs for this target.
+func (target *BuildTarget) AllBuildInputs() []BuildInput {
+	srcs := target.AllSources()
+	data := target.AllData()
+	tools := target.AllTools()
+
+	size := len(srcs) + len(data) + len(tools)
+	inputs := make([]BuildInput, 0, size)
+	inputs = append(inputs, srcs...)
+	inputs = append(inputs, data...)
+	inputs = append(inputs, tools...)
+	return inputs
+}
+
 // AllSources returns all the sources of this rule.
 func (target *BuildTarget) AllSources() []BuildInput {
 	if target.NamedSources == nil {
 		return target.Sources
 	}
-	return target.allBuildInputs(target.Sources, target.NamedSources)
+	return combinedBuildInputs(target.Sources, target.NamedSources)
 }
 
-func (target *BuildTarget) allBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput {
-	ret := unnamed
+// combinedBuildInputs combines the unnamed inputs and the values of named inputs into one slice.
+func combinedBuildInputs(unnamed []BuildInput, named map[string][]BuildInput) []BuildInput {
 	keys := make([]string, 0, len(named))
-	for k := range named {
+	size := 0
+	for k, vals := range named {
 		keys = append(keys, k)
+		size += len(vals)
 	}
 	sort.Strings(keys)
+
+	size += len(unnamed)
+	ret := make([]BuildInput, 0, size)
+	ret = append(ret, unnamed...)
 	for _, k := range keys {
 		ret = append(ret, named[k]...)
 	}
@@ -1710,7 +1730,7 @@ func (target *BuildTarget) AllData() []BuildInput {
 		return target.Data
 	}
 
-	return target.allBuildInputs(target.Data, target.NamedData)
+	return combinedBuildInputs(target.Data, target.NamedData)
 }
 
 // AllDebugData returns all the data for debugging this rule.
@@ -1721,7 +1741,7 @@ func (target *BuildTarget) AllDebugData() []BuildInput {
 	if target.Debug.namedData == nil {
 		return target.Debug.data
 	}
-	return target.allBuildInputs(target.Debug.data, target.Debug.namedData)
+	return combinedBuildInputs(target.Debug.data, target.Debug.namedData)
 }
 
 // DebugData returns unnamed data for debugging this rule.
@@ -1745,7 +1765,7 @@ func (target *BuildTarget) AllTools() []BuildInput {
 	if target.namedTools == nil {
 		return target.Tools
 	}
-	return target.allBuildInputs(target.Tools, target.namedTools)
+	return combinedBuildInputs(target.Tools, target.namedTools)
 }
 
 // ToolNames returns an ordered list of tool names.

src/core/graph.go

@@ -13,17 +13,6 @@ import (
 	"github.com/thought-machine/please/src/cmap"
 )
 
-type labelSet map[BuildLabel]struct{}
-
-func (ls labelSet) add(l BuildLabel) {
-	ls[l] = struct{}{}
-}
-
-func (ls labelSet) contains(l BuildLabel) bool {
-	_, ok := ls[l]
-	return ok
-}
-
 // A BuildGraph contains all the loaded targets and packages and maintains their
 // relationships, especially reverse dependencies which are calculated here.
 type BuildGraph struct {
@@ -35,7 +24,7 @@ type BuildGraph struct {
 	subrepos *cmap.Map[string, *Subrepo]
 	// Subincludes that are subincluded by other subincludes
 	subincludeSubincludes map[BuildLabel]labelSet
-	// Use a mutex as a labelSet isn't atomic. We need to guard against inserting as well as mutating the value.
+	// Use a mutex as a LabelSet isn't atomic. We need to guard against inserting as well as mutating the value.
 	subincMux sync.Mutex
 }
 
@@ -165,7 +154,7 @@ func (graph *BuildGraph) PackageMap() map[string]*Package {
 // NewGraph constructs and returns a new BuildGraph.
 func NewGraph() *BuildGraph {
 	g := &BuildGraph{
-		targets:               cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, hashBuildLabel),
+		targets:               cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, HashBuildLabel),
 		packages:              cmap.New[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey),
 		subrepos:              cmap.New[string, *Subrepo](cmap.SmallShardCount, cmap.XXHash),
 		subincludeSubincludes: map[BuildLabel]labelSet{},
@@ -197,10 +186,10 @@ func (graph *BuildGraph) TransitiveSubincludes(l BuildLabel) []BuildLabel {
 }
 
 func (graph *BuildGraph) findTransitiveSubincludes(label BuildLabel, includes labelSet) {
-	if includes.contains(label) {
+	if includes.Contains(label) {
 		return
 	}
-	includes.add(label)
+	includes.Add(label)
 	for l := range graph.subincludeSubincludes[label] {
 		graph.findTransitiveSubincludes(l, includes)
 	}
@@ -215,5 +204,5 @@ func (graph *BuildGraph) RegisterTransitiveSubinclude(from, to BuildLabel) {
 		incs = labelSet{}
 		graph.subincludeSubincludes[from] = incs
 	}
-	incs.add(to)
+	incs.Add(to)
 }

src/core/package.go

@@ -34,23 +34,46 @@ type Package struct {
 	targets map[string]*BuildTarget
 	// Set of output files from rules.
 	Outputs map[string]*BuildTarget
+	// Includes metadata from parsing the package BUILD file.
+	Metadata PackageMetadata
 	// Protects access to above
 	mutex sync.RWMutex
 }
 
-// NewPackage constructs a new package with the given name.
-func NewPackage(name string) *Package {
-	return NewPackageSubrepo(name, "")
+// PackageOptions is a functional option type for configuring a new Package.
+type PackageOptions func(*Package)
+
+// WithPackageSubrepo returns a PackageOptions that sets the subrepo name for a new Package.
+func WithPackageSubrepo(name string) PackageOptions {
+	return func(p *Package) {
+		p.SubrepoName = name
+	}
 }
 
-// NewPackageSubrepo constructs a new package with the given name and subrepo.
-func NewPackageSubrepo(name, subrepo string) *Package {
-	return &Package{
-		Name:        name,
-		SubrepoName: subrepo,
-		targets:     map[string]*BuildTarget{},
-		Outputs:     map[string]*BuildTarget{},
+// WithPackageMetadata returns a PackageOptions that enables tracking of
+// metadata (like statement positions and subinclude mappings) for the Package.
+// This is required for features like 'plz export'.
+func WithPackageMetadata() PackageOptions {
+	return func(p *Package) {
+		p.Metadata = newPackageMetadata()
+	}
+}
+
+// NewPackage constructs a new package with the given name, and enables additional features
+// given the PackageOptions provided.
+func NewPackage(name string, options ...PackageOptions) *Package {
+	pkg := &Package{
+		Name:    name,
+		targets: map[string]*BuildTarget{},
+		Outputs: map[string]*BuildTarget{},
+		// Defaults to noop to avoid storing metadata for most operations
+		Metadata: newNoopPackageMetadata(),
+	}
+
+	for _, option := range options {
+		option(pkg)
 	}
+	return pkg
 }
 
 // Target returns the target with the given name, or nil if this package doesn't have one.
@@ -109,7 +132,7 @@ func (pkg *Package) AllSubincludes(graph *BuildGraph) []BuildLabel {
 
 	for _, s := range pkg.Subincludes {
 		for _, inc := range append(graph.TransitiveSubincludes(s), s) {
-			includes.add(inc)
+			includes.Add(inc)
 		}
 	}