diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index 710e713a..6541f0f2 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -15,6 +15,7 @@ type Github struct { SyncSecrets bool `mapstructure:"sync-secrets"` OmitArchivedRepositories bool `mapstructure:"omit-archived-repositories"` DirectCollaboratorsOnly bool `mapstructure:"direct-collaborators-only"` + SyncLastActivity bool `mapstructure:"sync-last-activity"` } func (c *Github) findFieldByTag(tagValue string) (any, bool) { diff --git a/pkg/config/config.go b/pkg/config/config.go index 5c676fac..5a142d66 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -91,6 +91,18 @@ var ( field.WithDescription("Organization of your github app"), field.WithRequired(true), ) + + // syncLastActivity is hidden from this connector's GUI config and --help + // since it only applies to GitHub Enterprise audit-log access. + // baton-github-enterprise sets it directly on the shared Github struct, + // bypassing this CLI layer, so hiding it here doesn't affect that connector. + syncLastActivity = field.BoolField( + "sync-last-activity", + field.WithDisplayName("Sync users last activity"), + field.WithDescription("See when members were last active in your organizations."), + field.WithHidden(true), + field.WithExportTarget(field.ExportTargetCLIOnly), + ) ) //go:generate go run ./gen @@ -107,6 +119,7 @@ var Config = field.NewConfiguration( syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + syncLastActivity, }, field.WithConnectorDisplayName("GitHub v2"), field.WithHelpUrl("/docs/baton/github-v2"), diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index b38ed4e8..72f0bf07 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -127,6 +127,7 @@ type GitHub struct { omitArchivedRepositories bool directCollaboratorsOnly bool enterprises []string + syncLastActivity bool } func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { @@ -157,8 +158,18 @@ func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour return resourceSyncers } +func (gh *GitHub) EventFeeds(_ context.Context) []connectorbuilder.EventFeed { + if !gh.syncLastActivity { + return nil + } + + return []connectorbuilder.EventFeed{ + newUsageEventFeed(gh.client, gh.orgs), + } +} + // Metadata returns metadata about the connector. -func (gh *GitHub) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) { +func (gh *GitHub) Metadata(_ context.Context) (*v2.ConnectorMetadata, error) { return &v2.ConnectorMetadata{ DisplayName: "GitHub", AccountCreationSchema: &v2.ConnectorAccountCreationSchema{ @@ -346,6 +357,7 @@ func newWithGithubPAT(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, }, nil } @@ -452,6 +464,7 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, } return gh, nil } diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go new file mode 100644 index 00000000..efd24be5 --- /dev/null +++ b/pkg/connector/usage_event_feed.go @@ -0,0 +1,296 @@ +package connector + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/google/go-github/v69/github" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// defaultActivityLookback bounds the very first poll when no earliest-event +// boundary is given yet; later polls advance via the feed's own cursor. +const defaultActivityLookback = 1 * time.Hour + +// maxAuditLogPagesPerCall caps total pages walked across all orgs per call +// so one very active org can't stall the feed; remaining pages resume via the cursor. +const maxAuditLogPagesPerCall = 20 + +// usageEventFeed streams member activity from each org's audit log as usage +// events, since GitHub has no per-user "last activity" field to sync directly. +type usageEventFeed struct { + client *github.Client + orgs []string +} + +func newUsageEventFeed(client *github.Client, orgs []string) *usageEventFeed { + return &usageEventFeed{client: client, orgs: orgs} +} + +func (f *usageEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFeedMetadata { + return &v2.EventFeedMetadata{ + Id: "github_usage_event_feed", + SupportedEventTypes: []v2.EventType{v2.EventType_EVENT_TYPE_USAGE}, + } +} + +// usageEventPageToken tracks progress through one pass over every configured +// org's audit log, walked newest-first until an entry at or before Since is +// reached (already seen in a previous pass). +type usageEventPageToken struct { + Orgs []string `json:"orgs,omitempty"` + OrgIndex int `json:"org_index"` + AuditLogCursor string `json:"audit_log_cursor,omitempty"` + Since string `json:"since,omitempty"` +} + +func unmarshalUsageEventPageToken(pToken *pagination.StreamToken) (*usageEventPageToken, error) { + pt := &usageEventPageToken{} + if pToken == nil || pToken.Cursor == "" { + return pt, nil + } + data, err := base64.StdEncoding.DecodeString(pToken.Cursor) + if err != nil { + return nil, fmt.Errorf("baton-github: failed to decode usage event feed cursor: %w", err) + } + if err := json.Unmarshal(data, pt); err != nil { + return nil, fmt.Errorf("baton-github: failed to unmarshal usage event feed cursor: %w", err) + } + return pt, nil +} + +func (pt *usageEventPageToken) marshal() (string, error) { + data, err := json.Marshal(pt) + if err != nil { + return "", fmt.Errorf("baton-github: failed to marshal usage event feed cursor: %w", err) + } + return base64.StdEncoding.EncodeToString(data), nil +} + +func (f *usageEventFeed) ListEvents( + ctx context.Context, + earliestEvent *timestamppb.Timestamp, + pToken *pagination.StreamToken, +) ([]*v2.Event, *pagination.StreamState, annotations.Annotations, error) { + l := ctxzap.Extract(ctx) + + if f.client == nil { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + cursor, err := unmarshalUsageEventPageToken(pToken) + if err != nil { + return nil, nil, nil, err + } + + if len(cursor.Orgs) == 0 { + // Snapshot the org list and "since" boundary once per pass, so + // mid-pass config changes don't shift what gets walked. + orgs, err := getOrgs(ctx, f.client, f.orgs) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: failed to list orgs for usage event feed: %w", err) + } + if len(orgs) == 0 { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + since := time.Now().Add(-defaultActivityLookback) + // Guard against a zero/degenerate earliestEvent producing a + // nonsensical "since year 1" query that GitHub's search parser rejects. + if earliestEvent != nil { + if t := earliestEvent.AsTime(); !t.IsZero() && t.After(time.Unix(0, 0)) { + since = t + } + } + + cursor = &usageEventPageToken{ + Orgs: orgs, + Since: since.Format(time.RFC3339Nano), + } + } + + if cursor.OrgIndex < 0 || cursor.OrgIndex >= len(cursor.Orgs) { + cursor.OrgIndex = 0 + cursor.AuditLogCursor = "" + } + + since, err := time.Parse(time.RFC3339Nano, cursor.Since) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: invalid usage event feed cursor timestamp: %w", err) + } + // created:>= is sent server-side so GitHub excludes already-seen + // entries; the check below stays as a safety net in case it's ignored. + sincePhrase := "created:>=" + since.UTC().Format("2006-01-02T15:04:05-07:00") + + var events []*v2.Event + // Tightest (lowest Remaining) rate limit seen across this call's requests. + var tightestRateLimit *v2.RateLimitDescription + + // TODO(jdc): Probably change this for loop for a series of requests that uses a more complex pagination cursor. + for page := 0; page < maxAuditLogPagesPerCall; page++ { + orgName := cursor.Orgs[cursor.OrgIndex] + + opts := &github.GetAuditLogOptions{ + Order: github.Ptr("desc"), + // "web" excludes raw git-protocol events (push/fetch/clone), + // which dominate audit-log volume without losing members who are + // otherwise covered by their web/API activity. + Include: github.Ptr("web"), + Phrase: github.Ptr(sincePhrase), + ListCursorOptions: github.ListCursorOptions{ + PerPage: maxPageSize, + Page: cursor.AuditLogCursor, + }, + } + + entries, resp, err := f.client.Organizations.GetAuditLog(ctx, orgName, opts) + // Read rate-limit headers before the error branch nils resp, since a + // 429 still carries them. + if resp != nil { + if rl, rlErr := extractRateLimitData(resp); rlErr == nil { + if tightestRateLimit == nil || rl.GetRemaining() < tightestRateLimit.GetRemaining() { + tightestRateLimit = rl + } + } + } + if err != nil { + // Skip-and-continue only for permanent per-org conditions (no + // audit-log access); anything else aborts instead of wasting the + // rest of the page budget. Rate-limit checks come first since + // GitHub can signal rate limiting via a 403. + var rateLimitErr *github.RateLimitError + var abuseRateLimitErr *github.AbuseRateLimitError + retryable := errors.As(err, &rateLimitErr) || errors.As(err, &abuseRateLimitErr) || + isRatelimited(resp) || isTemporarilyUnavailable(resp) + + switch { + case retryable: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + case isNotFoundError(resp) || isPermissionError(resp): + l.Warn("org lacks audit-log access, skipping it for this pass", + zap.String("org", orgName), zap.Error(err)) + entries, resp = nil, nil + default: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + } + } + + reachedBoundary := false + for _, entry := range entries { + // Check every entry's timestamp, even filtered ones, so an all-bot page still stops pagination. + if ts := entry.GetTimestamp().Time; !ts.IsZero() && !ts.After(since) { + reachedBoundary = true + break + } + + evt, ok := usageEventFromAuditEntry(orgName, entry) + if !ok { + continue + } + events = append(events, evt) + } + + if resp != nil && resp.NextPageToken != "" && !reachedBoundary { + cursor.AuditLogCursor = resp.NextPageToken + continue + } + + // Done with this org for this pass - advance to the next one. + cursor.OrgIndex++ + cursor.AuditLogCursor = "" + if cursor.OrgIndex >= len(cursor.Orgs) { + // Pass complete - the next call gets a fresh earliestEvent, so + // nothing needs to survive in the cursor. + tokenStr, err := (&usageEventPageToken{}).marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: false}, annos, nil + } + } + + tokenStr, err := cursor.marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: true}, annos, nil +} + +// usageEventFromAuditEntry converts one audit-log entry into a usage event +// tying the actor to the org they acted in. Returns ok=false when the entry +// can't be attributed to a synced user resource. +func usageEventFromAuditEntry(orgName string, entry *github.AuditEntry) (*v2.Event, bool) { + actor := entry.GetActor() + actorID := entry.GetActorID() + ts := entry.GetTimestamp().Time + if actorID == 0 || ts.IsZero() { + return nil, false + } + + // actor_is_bot is real but undocumented (only in AdditionalFields); trust + // it when present, else fall back to the "[bot]" login suffix. Bots + // aren't synced as users, so their events wouldn't correlate to anything. + if isBot, ok := entry.AdditionalFields["actor_is_bot"].(bool); ok { + if isBot { + return nil, false + } + } else if strings.HasSuffix(actor, "[bot]") { + return nil, false + } + + orgID := entry.GetOrgID() + if orgID == 0 { + return nil, false + } + + id := entry.GetDocumentID() + if id == "" { + // No stable ID from GitHub - synthesize one so dedup doesn't collapse + // every entry missing _document_id into one event. + id = fmt.Sprintf("%d:%d:%d:%s", orgID, actorID, ts.UnixNano(), entry.GetAction()) + } + + return &v2.Event{ + Id: id, + OccurredAt: timestamppb.New(ts), + Event: &v2.Event_UsageEvent{ + UsageEvent: &v2.UsageEvent{ + TargetResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeOrg.Id, + Resource: strconv.FormatInt(orgID, 10), + }, + DisplayName: orgName, + }, + ActorResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeUser.Id, + Resource: strconv.FormatInt(actorID, 10), + }, + DisplayName: actor, + }, + }, + }, + }, true +} diff --git a/pkg/connector/usage_event_feed_test.go b/pkg/connector/usage_event_feed_test.go new file mode 100644 index 00000000..723c62c0 --- /dev/null +++ b/pkg/connector/usage_event_feed_test.go @@ -0,0 +1,536 @@ +package connector + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/google/go-github/v69/github" + "github.com/migueleliasweb/go-github-mock/src/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestUsageEventFromAuditEntry(t *testing.T) { + ts := time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + entry *github.AuditEntry + ok bool + }{ + { + name: "valid entry", + entry: &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + }, + ok: true, + }, + { + name: "missing actor id", + entry: &github.AuditEntry{ + Actor: github.Ptr("octocat"), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + }, + ok: false, + }, + { + name: "missing org id", + entry: &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + Timestamp: &github.Timestamp{Time: ts}, + }, + ok: false, + }, + { + name: "missing timestamp", + entry: &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + }, + ok: false, + }, + { + name: "bot actor by login suffix", + entry: &github.AuditEntry{ + Actor: github.Ptr("dependabot[bot]"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + }, + ok: false, + }, + { + name: "bot actor by actor_is_bot field", + entry: &github.AuditEntry{ + Actor: github.Ptr("some-app-installation"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + AdditionalFields: map[string]interface{}{"actor_is_bot": true}, + }, + ok: false, + }, + { + name: "actor_is_bot false overrides a non-matching suffix check", + entry: &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + AdditionalFields: map[string]interface{}{"actor_is_bot": false}, + }, + ok: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + evt, ok := usageEventFromAuditEntry("octo-org", tt.entry) + require.Equal(t, tt.ok, ok) + if !tt.ok { + return + } + require.Equal(t, ts, evt.GetOccurredAt().AsTime()) + require.Equal(t, "123", evt.GetUsageEvent().GetActorResource().GetId().GetResource()) + require.Equal(t, "456", evt.GetUsageEvent().GetTargetResource().GetId().GetResource()) + require.Equal(t, resourceTypeUser.Id, evt.GetUsageEvent().GetActorResource().GetId().GetResourceType()) + require.Equal(t, resourceTypeOrg.Id, evt.GetUsageEvent().GetTargetResource().GetId().GetResourceType()) + }) + } +} + +func TestUsageEventFromAuditEntry_IdFallback(t *testing.T) { + ts := time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + + t.Run("uses the real document id when present", func(t *testing.T) { + entry := &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + DocumentID: github.Ptr("real-doc-id"), + Action: github.Ptr("repo.create"), + } + evt, ok := usageEventFromAuditEntry("octo-org", entry) + require.True(t, ok) + require.Equal(t, "real-doc-id", evt.GetId()) + }) + + t.Run("synthesizes a stable id when the document id is missing", func(t *testing.T) { + entry := &github.AuditEntry{ + Actor: github.Ptr("octocat"), + ActorID: github.Ptr(int64(123)), + OrgID: github.Ptr(int64(456)), + Timestamp: &github.Timestamp{Time: ts}, + Action: github.Ptr("repo.create"), + } + evt, ok := usageEventFromAuditEntry("octo-org", entry) + require.True(t, ok) + require.Equal(t, fmt.Sprintf("456:123:%d:repo.create", ts.UnixNano()), evt.GetId()) + require.NotEmpty(t, evt.GetId()) + }) +} + +func TestUsageEventFeed_ListEvents_GracefulDegradation(t *testing.T) { + ctx := context.Background() + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, nil) + require.NoError(t, err) + require.Empty(t, events) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_SkipsOrgOn404(t *testing.T) { + ctx := context.Background() + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, nil) + require.NoError(t, err) + require.Empty(t, events) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_AbortsOnServerError(t *testing.T) { + ctx := context.Background() + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, nil) + require.Error(t, err, "a 5xx should abort the call, not be swallowed as a successful empty pass") + require.Nil(t, events) + require.Nil(t, state) +} + +func TestUsageEventFeed_ListEvents_AbortsOnRateLimit(t *testing.T) { + ctx := context.Background() + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Ratelimit-Remaining", "0") + w.WriteHeader(http.StatusForbidden) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, nil) + require.Error(t, err, "a rate-limited 403 (Remaining: 0) should abort, not be treated as a permission error") + require.Nil(t, events) + require.Nil(t, state) +} + +func TestUsageEventFeed_ListEvents_FiltersToSinceBoundary(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer1 := since.Add(2 * time.Hour) + newer2 := since.Add(1 * time.Hour) + older := since.Add(-1 * time.Hour) + + // Entries in descending order, as requested (Order: "desc"). + entries := []*github.AuditEntry{ + {Actor: github.Ptr("alice"), ActorID: github.Ptr(int64(1)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer1}}, + {Actor: github.Ptr("bob"), ActorID: github.Ptr(int64(2)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer2}}, + {Actor: github.Ptr("carol"), ActorID: github.Ptr(int64(3)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: older}}, + } + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatch(mock.GetOrgsAuditLogByOrg, entries), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, timestamppb.New(since), nil) + require.NoError(t, err) + require.Len(t, events, 2) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_ZeroEarliestEventFallsBackToDefaultLookback(t *testing.T) { + ctx := context.Background() + + var gotPhrase string + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPhrase = r.URL.Query().Get("phrase") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("[]")) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + // A zero timestamppb.Timestamp mirrors a degenerate caller-supplied + // start-at, which must not be trusted as a real boundary. + events, state, _, err := f.ListEvents(ctx, ×tamppb.Timestamp{}, nil) + require.NoError(t, err) + require.Empty(t, events) + require.False(t, state.HasMore) + require.NotContains(t, gotPhrase, "0001-01-01") + require.Contains(t, gotPhrase, "created:>=") +} + +func TestUsageEventFeed_ListEvents_ContinuesPastAnAllFilteredPage(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := since.Add(1 * time.Hour) + + // Page 1 is entirely bot activity - every entry gets filtered out by + // usageEventFromAuditEntry, so no event is ever appended on this page. + page1 := []*github.AuditEntry{ + {Actor: github.Ptr("dependabot[bot]"), ActorID: github.Ptr(int64(1)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + } + page2 := []*github.AuditEntry{ + {Actor: github.Ptr("octocat"), ActorID: github.Ptr(int64(2)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + } + + calls := 0 + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + if calls == 1 { + w.Header().Set("Link", `; rel="next"`) + _, _ = w.Write(mock.MustMarshal(page1)) + return + } + _, _ = w.Write(mock.MustMarshal(page2)) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, timestamppb.New(since), nil) + require.NoError(t, err) + require.Equal(t, 2, calls, "an all-filtered page must not be mistaken for the since boundary") + require.Len(t, events, 1) + require.Equal(t, "2", events[0].GetUsageEvent().GetActorResource().GetId().GetResource()) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_ReturnsTightestRateLimit(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := since.Add(1 * time.Hour) + + page1 := []*github.AuditEntry{ + {Actor: github.Ptr("octocat"), ActorID: github.Ptr(int64(1)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + } + page2 := []*github.AuditEntry{ + {Actor: github.Ptr("alice"), ActorID: github.Ptr(int64(2)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + } + + calls := 0 + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{{Login: github.Ptr("octo-org")}}), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Ratelimit-Limit", "1750") + if calls == 1 { + // First call reports plenty of budget left. + w.Header().Set("X-Ratelimit-Remaining", "500") + w.Header().Set("Link", `; rel="next"`) + _, _ = w.Write(mock.MustMarshal(page1)) + return + } + // Second call is the tighter one - this is the value that + // should win. + w.Header().Set("X-Ratelimit-Remaining", "10") + _, _ = w.Write(mock.MustMarshal(page2)) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, _, annos, err := f.ListEvents(ctx, timestamppb.New(since), nil) + require.NoError(t, err) + require.Equal(t, 2, calls) + require.Len(t, events, 2) + + var rl v2.RateLimitDescription + found, err := annos.Pick(&rl) + require.NoError(t, err) + require.True(t, found, "expected a rate-limit annotation to be returned") + require.Equal(t, int64(10), rl.GetRemaining()) + require.Equal(t, int64(1750), rl.GetLimit()) +} + +func TestUsageEventFeed_ListEvents_AdvancesAcrossMultipleOrgs(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := since.Add(1 * time.Hour) + + entriesByOrg := map[string][]*github.AuditEntry{ + "octo-org-a": { + {Actor: github.Ptr("alice"), ActorID: github.Ptr(int64(1)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + }, + "octo-org-b": { + {Actor: github.Ptr("bob"), ActorID: github.Ptr(int64(2)), OrgID: github.Ptr(int64(8)), Timestamp: &github.Timestamp{Time: newer}}, + }, + } + + var seenOrgs []string + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{ + {Login: github.Ptr("octo-org-a")}, + {Login: github.Ptr("octo-org-b")}, + }), + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + org := strings.Split(r.URL.Path, "/")[2] + seenOrgs = append(seenOrgs, org) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(mock.MustMarshal(entriesByOrg[org])) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, timestamppb.New(since), nil) + require.NoError(t, err) + require.False(t, state.HasMore) + require.Equal(t, []string{"octo-org-a", "octo-org-b"}, seenOrgs, "should walk both orgs, in order, within one call") + require.Len(t, events, 2) + + actorIDs := []string{ + events[0].GetUsageEvent().GetActorResource().GetId().GetResource(), + events[1].GetUsageEvent().GetActorResource().GetId().GetResource(), + } + require.ElementsMatch(t, []string{"1", "2"}, actorIDs) +} + +func TestUsageEventFeed_ListEvents_ResumesFromPersistedCursor(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := since.Add(1 * time.Hour) + + // Simulate a previous call that already finished "octo-org-a" and was + // mid-page through "octo-org-b" with its own audit-log cursor. + resumeToken := &usageEventPageToken{ + Orgs: []string{"octo-org-a", "octo-org-b"}, + OrgIndex: 1, + AuditLogCursor: "existing-cursor", + Since: since.Format(time.RFC3339), + } + cursorStr, err := resumeToken.marshal() + require.NoError(t, err) + + var gotOrg, gotPage string + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotOrg = strings.Split(r.URL.Path, "/")[2] + gotPage = r.URL.Query().Get("page") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(mock.MustMarshal([]*github.AuditEntry{ + {Actor: github.Ptr("bob"), ActorID: github.Ptr(int64(2)), OrgID: github.Ptr(int64(8)), Timestamp: &github.Timestamp{Time: newer}}, + })) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, &pagination.StreamToken{Cursor: cursorStr}) + require.NoError(t, err) + require.Equal(t, "octo-org-b", gotOrg, "should resume at the persisted org, not restart from octo-org-a") + require.Equal(t, "existing-cursor", gotPage, "should resume with the persisted audit-log cursor") + require.Len(t, events, 1) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_RecoversFromOutOfBoundsCursor(t *testing.T) { + ctx := context.Background() + + since := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := since.Add(1 * time.Hour) + + // A corrupted/stale cursor: OrgIndex points past the end of Orgs. + badToken := &usageEventPageToken{ + Orgs: []string{"octo-org"}, + OrgIndex: 5, + AuditLogCursor: "stale-cursor", + Since: since.Format(time.RFC3339Nano), + } + cursorStr, err := badToken.marshal() + require.NoError(t, err) + + var gotPage string + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatchHandler( + mock.GetOrgsAuditLogByOrg, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPage = r.URL.Query().Get("page") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(mock.MustMarshal([]*github.AuditEntry{ + {Actor: github.Ptr("octocat"), ActorID: github.Ptr(int64(1)), OrgID: github.Ptr(int64(9)), Timestamp: &github.Timestamp{Time: newer}}, + })) + }), + ), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + require.NotPanics(t, func() { + events, state, _, err := f.ListEvents(ctx, nil, &pagination.StreamToken{Cursor: cursorStr}) + require.NoError(t, err) + require.Len(t, events, 1) + require.False(t, state.HasMore) + }) + require.Empty(t, gotPage, "should discard the stale per-org cursor when OrgIndex is reset") +} + +func TestUsageEventFeed_ListEvents_NoOrgs(t *testing.T) { + ctx := context.Background() + + httpClient := mock.NewMockedHTTPClient( + mock.WithRequestMatch(mock.GetUserOrgs, []*github.Organization{}), + ) + + f := newUsageEventFeed(github.NewClient(httpClient), nil) + + events, state, _, err := f.ListEvents(ctx, nil, nil) + require.NoError(t, err) + require.Empty(t, events) + require.False(t, state.HasMore) +} + +func TestUsageEventFeed_ListEvents_NilClient(t *testing.T) { + ctx := context.Background() + + f := newUsageEventFeed(nil, nil) + + events, state, _, err := f.ListEvents(ctx, nil, &pagination.StreamToken{}) + require.NoError(t, err) + require.Empty(t, events) + require.False(t, state.HasMore) +}