From aa3ee293698bab497f0382b1fd65f480f95ca245 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 00:50:12 +0000 Subject: [PATCH] Take people in from a social client, and assemble an OpenProfile.md for each POST /api/v1/people/from-social opens a person from a network handle (the accounts myna follows, or their followers), puts them in a campaign, keeps the bio as a content_topic signal so a card can trigger, and queues one `openprofile` job per person. The job reads the network's public profile (Bluesky AppView, Mastodon lookup, or the page's OpenGraph card), follows the home page it names for its card and rel=me links, and looks for a published /.well-known/openprofile.md. A rel=me back to the profile is two sources agreeing: the person goes to 0.9 and the site's accounts become identities, a mailto among them included. Otherwise 0.5, and the links are only links. The Markdown follows logicsrc.com/openprofile and is served at GET /api/v1/people/:id/openprofile.md as text/markdown, workspace-scoped. `og add-social` and `og openprofile` on the CLI; `add_people_from_social` and `get_openprofile` on the MCP server. Migration 0035 adds `openprofiles`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S7yeJUHGxA4P5N74xnsRPQ --- apps/api/src/app.ts | 182 +++++++- apps/api/src/social-intake.test.ts | 134 ++++++ apps/cli/src/commands.ts | 89 ++++ apps/mcp/src/tools.ts | 76 +++ apps/server/src/index.ts | 15 + migrations/0035_openprofiles.sql | 25 + packages/pipeline/src/index.ts | 11 + packages/pipeline/src/jobs.ts | 9 + packages/pipeline/src/openprofile.ts | 404 ++++++++++++++++ packages/pipeline/src/social-intake.test.ts | 402 ++++++++++++++++ packages/pipeline/src/social-intake.ts | 357 +++++++++++++++ packages/providers/src/index.ts | 16 + packages/providers/src/site/extract.ts | 10 +- .../providers/src/site/openprofile.test.ts | 276 +++++++++++ packages/providers/src/site/openprofile.ts | 433 ++++++++++++++++++ 15 files changed, 2417 insertions(+), 22 deletions(-) create mode 100644 apps/api/src/social-intake.test.ts create mode 100644 migrations/0035_openprofiles.sql create mode 100644 packages/pipeline/src/openprofile.ts create mode 100644 packages/pipeline/src/social-intake.test.ts create mode 100644 packages/pipeline/src/social-intake.ts create mode 100644 packages/providers/src/site/openprofile.test.ts create mode 100644 packages/providers/src/site/openprofile.ts diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index cec3bc5..44b8540 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -84,6 +84,7 @@ import { crawlDedupeKey, startContactImport, importContactChunk, + intakeSocialPeople, finishContactImport, createCadence, createRule, @@ -176,6 +177,8 @@ import { /** One paste, one reviewable unit of work. */ const MAX_BULK_URLS = 100; +/** People per social intake request. myna sends a follow list in pages of this size. */ +const MAX_SOCIAL_INTAKE = 200; export interface AppOptions { readonly db: Client; @@ -1705,6 +1708,127 @@ export function createApp(options: AppOptions): Hono { return c.json({ people: rows.rows }); }); + /** + * People handed over from a social client: a handle, a display name, a bio + * and a profile URL, and nothing more. myna sends the accounts it follows + * (or their followers) here so OutreachGraph can decide whether each is + * worth an offer. Every person lands in a campaign, the bio becomes a + * signal, and an `openprofile` job reads what their profile and home page + * say, so a card can appear once there is something to act on. + * + * 202 rather than 201 because the person exists but the assessment has not + * run: the profile read and the recommendation follow in the worker. + */ + api.post('/people/from-social', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + + if (!canApprove(actor)) throw ApiError.forbidden('adding people'); + + const raw = safeJson(await c.req.raw.text()); + const submitted = Array.isArray(raw.people) + ? raw.people + : raw.handle !== undefined + ? [raw] + : []; + if (!submitted.length) + throw ApiError.badRequest('people is required: [{ network, handle, ... }]'); + if (submitted.length > MAX_SOCIAL_INTAKE) { + throw ApiError.badRequest(`at most ${MAX_SOCIAL_INTAKE} people per request`); + } + + let campaignId = + typeof raw.campaignId === 'string' && raw.campaignId.trim() + ? raw.campaignId.trim() + : undefined; + if (campaignId) { + const owned = await queryOne<{ id: string }>( + db, + `SELECT id FROM campaigns WHERE id = ? AND workspace_id = ? AND status != 'archived'`, + [campaignId, actor.workspaceId], + ); + if (!owned) throw ApiError.notFound('campaign'); + } else { + campaignId = await ensureDefaultCampaign(db, actor.workspaceId); + } + + const source = + typeof raw.source === 'string' && raw.source.trim() ? raw.source.trim().slice(0, 40) : 'api'; + const text = (entry: Record, key: string): string | undefined => + typeof entry[key] === 'string' ? (entry[key] as string) : undefined; + const result = await intakeSocialPeople( + { db }, + { + workspaceId: actor.workspaceId, + campaignId, + source, + people: (submitted as Record[]).map((entry) => ({ + network: String(entry.network ?? ''), + handle: String(entry.handle ?? ''), + profileUrl: text(entry, 'profileUrl'), + platformUserId: text(entry, 'platformUserId'), + displayName: text(entry, 'displayName'), + bio: text(entry, 'bio'), + avatarUrl: text(entry, 'avatarUrl'), + followers: typeof entry.followers === 'number' ? entry.followers : undefined, + via: text(entry, 'via'), + })), + }, + ); + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: 'people.social_intake', + entityKind: 'campaign', + entityId: campaignId, + detail: { + source, + created: result.created, + existing: result.existing, + queued: result.queued, + rejected: result.rejected.length, + }, + }); + + return c.json({ campaignId, ...result }, 202); + }); + + /** + * The OpenProfile.md OutreachGraph assembled for a person, or the one they + * publish themselves. Workspace-scoped like everything else about a person: + * a reader has to hold the person in a campaign of their own. + */ + api.get('/people/:id/openprofile.md', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + const personId = c.req.param('id'); + + const held = await queryOne<{ person_id: string }>( + db, + `SELECT cp.person_id FROM campaign_people cp JOIN people p ON p.id = cp.person_id + WHERE cp.person_id = ? AND cp.workspace_id = ? AND p.status != 'deleted' LIMIT 1`, + [personId, actor.workspaceId], + ); + if (!held) throw ApiError.notFound('person'); + + const profile = await queryOne<{ + markdown: string; + generated_at: string; + published_url: string | null; + }>(db, 'SELECT markdown, generated_at, published_url FROM openprofiles WHERE person_id = ?', [ + personId, + ]); + if (!profile) throw ApiError.notFound('openprofile'); + + return c.body(profile.markdown, 200, { + 'content-type': 'text/markdown; charset=utf-8', + 'last-modified': new Date(profile.generated_at).toUTCString(), + ...(profile.published_url ? { link: `<${profile.published_url}>; rel="canonical"` } : {}), + }); + }); + api.get('/people/:id', async (c) => { const actor = c.get('actor'); const db = c.get('db'); @@ -1713,30 +1837,42 @@ export function createApp(options: AppOptions): Hono { const person = await repo.getPerson(db, personId); if (!person || person.status === 'deleted') throw ApiError.notFound('person'); - const [identities, companyIdentities, signals, provenance, emailCandidates, membership] = - await Promise.all([ - repo.listIdentities(db, personId), - repo.listCompanyIdentities(db, personId), - repo.listPersonSignals(db, actor.workspaceId, personId), - repo.listProvenance(db, personId), - // Served here rather than behind its own fetch: deciding on an address - // is a judgement about this person, made with their evidence on screen. - candidatesForPerson(db, actor.workspaceId, personId), - // Which campaign this person belongs to, so anything acting on them - // acts within it. Without this a caller has to guess, and a wrong - // guess enrols somebody into a campaign they were never part of — - // which then scores and drafts for them against the wrong brief. - queryOne<{ campaign_id: string }>( - db, - `SELECT cp.campaign_id + const [ + identities, + companyIdentities, + signals, + provenance, + emailCandidates, + membership, + openprofile, + ] = await Promise.all([ + repo.listIdentities(db, personId), + repo.listCompanyIdentities(db, personId), + repo.listPersonSignals(db, actor.workspaceId, personId), + repo.listProvenance(db, personId), + // Served here rather than behind its own fetch: deciding on an address + // is a judgement about this person, made with their evidence on screen. + candidatesForPerson(db, actor.workspaceId, personId), + // Which campaign this person belongs to, so anything acting on them + // acts within it. Without this a caller has to guess, and a wrong + // guess enrols somebody into a campaign they were never part of — + // which then scores and drafts for them against the wrong brief. + queryOne<{ campaign_id: string }>( + db, + `SELECT cp.campaign_id FROM campaign_people cp JOIN campaigns c ON c.id = cp.campaign_id WHERE cp.person_id = ? AND c.workspace_id = ? AND c.status != 'archived' ORDER BY cp.updated_at DESC LIMIT 1`, - [personId, actor.workspaceId], - ), - ]); + [personId, actor.workspaceId], + ), + queryOne<{ generated_at: string; published_url: string | null }>( + db, + 'SELECT generated_at, published_url FROM openprofiles WHERE person_id = ?', + [personId], + ), + ]); return c.json({ person, @@ -1746,6 +1882,14 @@ export function createApp(options: AppOptions): Hono { signals, provenance, emailCandidates, + // Where the assembled OpenProfile.md is served, once the job has run. + openprofile: openprofile + ? { + url: `/api/v1/people/${personId}/openprofile.md`, + generatedAt: openprofile.generated_at, + publishedUrl: openprofile.published_url, + } + : null, }); }); diff --git a/apps/api/src/social-intake.test.ts b/apps/api/src/social-intake.test.ts new file mode 100644 index 0000000..46eaeaf --- /dev/null +++ b/apps/api/src/social-intake.test.ts @@ -0,0 +1,134 @@ +/** + * The social intake route and the OpenProfile endpoint. + * + * What has to hold: a handle becomes a person in the default campaign with + * an openprofile job queued, junk is named and the rest lands, a flat single + * person is accepted, a campaign from another workspace is refused, and the + * assembled Markdown is served as text/markdown only to a workspace that + * holds the person. + */ + +import { afterEach, describe, expect, test } from 'bun:test'; +import type { Hono } from 'hono'; +import { createApp } from './app'; +import type { AppEnv, RequestActor } from './context'; +import { seedDatabase, SEED, type SeededDatabase } from './test-seed'; + +const ACTOR: RequestActor = { + userId: SEED.userId, + workspaceId: SEED.workspaceId, + organizationId: SEED.organizationId, + role: 'owner', +}; + +let active: SeededDatabase | undefined; + +afterEach(() => { + active?.cleanup(); + active = undefined; +}); + +async function harness(label: string): Promise<{ app: Hono; seeded: SeededDatabase }> { + const seeded = await seedDatabase(label); + active = seeded; + const app = createApp({ db: seeded.db, authenticate: async () => ACTOR }); + return { app, seeded }; +} + +const get = (app: Hono, path: string) => app.request(`/api/v1${path}`); +const post = (app: Hono, path: string, body: unknown = {}) => + app.request(`/api/v1${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + +describe('POST /people/from-social', () => { + test('a handle becomes a person in the default campaign with an openprofile job queued', async () => { + const { app, seeded } = await harness('social-intake'); + const response = await post(app, '/people/from-social', { + source: 'myna', + people: [ + { + network: 'bluesky', + handle: '@ada.example', + displayName: 'Ada', + bio: 'Engines.', + via: 'following', + }, + { network: 'tiktok', handle: 'nope' }, + ], + }); + expect(response.status).toBe(202); + const body = await response.json(); + expect(body).toMatchObject({ campaignId: SEED.campaignId, created: 1, existing: 0, queued: 1 }); + expect(body.rejected).toEqual([{ handle: 'nope', reason: 'unknown network tiktok' }]); + expect(body.people[0]).toMatchObject({ + network: 'bluesky', + handle: 'ada.example', + created: true, + }); + + const listed = await (await get(app, '/people')).json(); + expect(listed.people.map((person: { id: string }) => person.id)).toContain(body.people[0].id); + + const job = await seeded.db.execute({ + sql: 'SELECT kind FROM jobs WHERE dedupe_key = ?', + args: [`openprofile:${body.people[0].id}`], + }); + expect(job.rows).toHaveLength(1); + + // No profile yet: the job has not run. + expect((await get(app, `/people/${body.people[0].id}/openprofile.md`)).status).toBe(404); + const detail = await (await get(app, `/people/${body.people[0].id}`)).json(); + expect(detail.openprofile).toBeNull(); + }); + + test('a single person may be sent flat; a foreign campaign and an empty body are refused', async () => { + const { app } = await harness('social-intake-flat'); + const flat = await post(app, '/people/from-social', { + network: 'mastodon', + handle: 'ada@hachyderm.io', + }); + expect(flat.status).toBe(202); + expect((await flat.json()).people[0]).toMatchObject({ + network: 'mastodon', + handle: 'ada@hachyderm.io', + }); + + const foreign = await post(app, '/people/from-social', { + campaignId: 'cmp_elsewhere', + people: [{ network: 'x', handle: 'ada' }], + }); + expect(foreign.status).toBe(404); + + expect((await post(app, '/people/from-social', {})).status).toBe(400); + }); +}); + +describe('GET /people/:id/openprofile.md', () => { + test('serves the assembled Markdown to a workspace that holds the person', async () => { + const { app, seeded } = await harness('social-intake-openprofile'); + await seeded.db.execute({ + sql: `INSERT INTO openprofiles (person_id, markdown, sources_json, published_url, generated_at) + VALUES (?, ?, '[]', 'https://jane.example/.well-known/openprofile.md', ?)`, + args: [SEED.personId, '# Jane\n\n- **Kind**: person\n', '2026-09-13T00:00:00.000Z'], + }); + const response = await get(app, `/people/${SEED.personId}/openprofile.md`); + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toBe('text/markdown; charset=utf-8'); + expect(response.headers.get('link')).toBe( + '; rel="canonical"', + ); + expect(await response.text()).toBe('# Jane\n\n- **Kind**: person\n'); + + const detail = await (await get(app, `/people/${SEED.personId}`)).json(); + expect(detail.openprofile).toEqual({ + url: `/api/v1/people/${SEED.personId}/openprofile.md`, + generatedAt: '2026-09-13T00:00:00.000Z', + publishedUrl: 'https://jane.example/.well-known/openprofile.md', + }); + + expect((await get(app, '/people/per_nobody/openprofile.md')).status).toBe(404); + }); +}); diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts index ef77a15..285e936 100644 --- a/apps/cli/src/commands.ts +++ b/apps/cli/src/commands.ts @@ -47,6 +47,34 @@ function pad(value: string, width: number): string { return value.length >= width ? value : value + ' '.repeat(width - value.length); } +/** The network a profile URL belongs to, for the handful `og add-social` accepts by URL. */ +function networkFromUrl(url: string): string | undefined { + const host = new URL(url).hostname.replace(/^www\./, ''); + if (/(^|\.)bsky\.app$/.test(host)) return 'bluesky'; + if (/(^|\.)(x|twitter)\.com$/.test(host)) return 'x'; + if (/(^|\.)github\.com$/.test(host)) return 'github'; + if (/(^|\.)linkedin\.com$/.test(host)) return 'linkedin'; + if (/(^|\.)reddit\.com$/.test(host)) return 'reddit'; + if (/(^|\.)youtube\.com$/.test(host)) return 'youtube'; + if (/(^|\.)instagram\.com$/.test(host)) return 'instagram'; + // `/@user` on any other host is read as a Fediverse account. + return /^\/@[^/]+/.test(new URL(url).pathname) ? 'mastodon' : undefined; +} + +/** `https://bsky.app/profile/ada.example` → `ada.example`; `https://hachyderm.io/@ada` → `ada@hachyderm.io`. */ +function handleFromUrl(url: string): string | undefined { + const parsed = new URL(url); + const segments = parsed.pathname.split('/').filter(Boolean); + const first = segments[0]; + if (!first) return undefined; + if (first.startsWith('@')) { + const user = first.slice(1); + return user.includes('@') ? user : `${user}@${parsed.hostname}`; + } + const nested = /^(profile|in|user|u|c|channel)$/i.test(first) ? segments[1] : first; + return nested?.replace(/^@/, ''); +} + export const COMMANDS: readonly Command[] = [ { name: 'today', @@ -111,6 +139,67 @@ export const COMMANDS: readonly Command[] = [ return `Added ${text(result, 'personId', text(result, 'id', url))}`; }, }, + { + name: 'add-social', + usage: 'og add-social ... [--campaign ] [--via ]', + summary: 'Hand over people from a social network, for assessment and an OpenProfile.', + async run({ client, args, flags }) { + if (args.length === 0) { + throw new Error('at least one person is required: og add-social bluesky:ada.example'); + } + const people = args.map((entry) => { + const url = /^https?:\/\//i.test(entry) ? entry : undefined; + const colon = entry.indexOf(':'); + const network = url ? networkFromUrl(url) : colon > 0 ? entry.slice(0, colon) : undefined; + const handle = url ? handleFromUrl(url) : colon > 0 ? entry.slice(colon + 1) : entry; + if (!network || !handle) + throw new Error(`cannot place ${entry}: use network:handle or a profile url`); + return { + network, + handle, + ...(url ? { profileUrl: url } : {}), + ...(flagString(flags, 'via') ? { via: flagString(flags, 'via') } : {}), + }; + }); + + const result = (await client.post('/people/from-social', { + people, + source: 'og', + ...(flagString(flags, 'campaign') ? { campaignId: flagString(flags, 'campaign') } : {}), + })) as Record; + + const added = rows(result, 'people'); + const rejected = rows(result, 'rejected'); + const lines = added.map((person) => + [ + pad(text(person, 'id'), 30), + pad(text(person, 'created') === 'true' ? 'new' : 'known', 6), + `${text(person, 'network')}:${text(person, 'handle')}`, + ].join(' '), + ); + for (const entry of rejected) + lines.push(`rejected ${text(entry, 'handle')}: ${text(entry, 'reason')}`); + lines.push( + `${text(result, 'created', '0')} new, ${text(result, 'existing', '0')} known, ` + + `${text(result, 'queued', '0')} queued for an OpenProfile in campaign ${text(result, 'campaignId')}`, + ); + return lines.join('\n'); + }, + }, + { + name: 'openprofile', + usage: 'og openprofile ', + summary: 'The OpenProfile.md assembled for one person.', + async run({ client, args }) { + const personId = args[0]; + if (!personId) throw new Error('a person id is required: og openprofile '); + const result = (await client.get( + `/people/${encodeURIComponent(personId)}/openprofile.md`, + )) as Record; + // The route answers text/markdown; the client hands non-JSON back under `raw`. + return text(result, 'raw').trimEnd(); + }, + }, { name: 'signals', usage: 'og signals ', diff --git a/apps/mcp/src/tools.ts b/apps/mcp/src/tools.ts index 1ad8c94..c12a2a7 100644 --- a/apps/mcp/src/tools.ts +++ b/apps/mcp/src/tools.ts @@ -266,6 +266,82 @@ export const TOOLS: readonly ToolDefinition[] = [ ...(str(args, 'campaignId') ? { campaignId: str(args, 'campaignId') } : {}), }), }, + { + name: 'add_people_from_social', + title: 'Hand over people from a social network', + description: + 'Add people known only by a social handle (Bluesky, Mastodon, X, GitHub, ...): the accounts ' + + 'you follow, or their followers. Each lands in a campaign for assessment; their bio becomes a ' + + 'signal and an OpenProfile.md is assembled from their profile and home page. Nothing is sent.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { + people: { + type: 'array', + maxItems: 200, + items: { + type: 'object', + properties: { + network: { + type: 'string', + description: + 'bluesky, mastodon, x, github, reddit, youtube, instagram, linkedin, nostr', + }, + handle: { + type: 'string', + description: 'Without the @. A Fediverse handle keeps its host: ada@hachyderm.io.', + }, + profileUrl: { type: 'string' }, + platformUserId: { + type: 'string', + description: 'A DID or numeric id, when the network has one.', + }, + displayName: { type: 'string' }, + bio: { type: 'string' }, + avatarUrl: { type: 'string' }, + via: { + type: 'string', + description: 'How you came by them: follow, following, followers, graph.', + }, + }, + required: ['network', 'handle'], + }, + }, + campaignId: { type: 'string', description: "Defaults to the workspace's active campaign." }, + source: { + type: 'string', + description: 'The client sending them, recorded on the audit trail.', + }, + }, + required: ['people'], + }, + run: (client, args) => + client.post('/people/from-social', { + people: Array.isArray(args.people) ? args.people : [], + source: str(args, 'source') ?? 'mcp', + ...(str(args, 'campaignId') ? { campaignId: str(args, 'campaignId') } : {}), + }), + }, + { + name: 'get_openprofile', + title: "Get a person's OpenProfile.md", + description: + 'The OpenProfile.md (logicsrc.com/openprofile) assembled for one person: name, handle, home ' + + 'page, the accounts that are theirs, topics. Absent until the openprofile job has run for them.', + readOnly: true, + inputSchema: { + type: 'object', + properties: { personId: { type: 'string' } }, + required: ['personId'], + }, + run: async (client, args) => { + const result = (await client.get( + `/people/${encodeURIComponent(require(args, 'personId'))}/openprofile.md`, + )) as { raw?: string }; + return { markdown: result.raw ?? '' }; + }, + }, { name: 'suppress', title: 'Never contact this person again', diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 42430d9..fce5620 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -54,6 +54,7 @@ import { runCadences, runCrawlJob, runDiscoveryJob, + runOpenProfileJob, loadImapCredentials, receiveReplies, runListening, @@ -612,6 +613,20 @@ async function runJob(job: QueuedJob): Promise { await processDeletion(db, deletionJobId); return; } + case 'openprofile': { + const result = await runOpenProfileJob({ db }, job); + console.log( + `openprofile ${result.personId}: ${result.outcome}` + + (result.outcome === 'ok' || result.outcome === 'published' + ? `, ${result.accounts} accounts, ${result.identities} new identit${result.identities === 1 ? 'y' : 'ies'}` + + (result.corroborated ? ', corroborated by rel=me' : '') + + (result.recommendationId ? `, card ${result.recommendationId}` : ', no card yet') + : result.detail + ? ` (${result.detail})` + : ''), + ); + return; + } case 'enrich_contact': { const { personId } = job.payload as { personId?: string }; if (!personId) throw new Error('enrich_contact needs personId'); diff --git a/migrations/0035_openprofiles.sql b/migrations/0035_openprofiles.sql new file mode 100644 index 0000000..ef5e3de --- /dev/null +++ b/migrations/0035_openprofiles.sql @@ -0,0 +1,25 @@ +-- 0035_openprofiles.sql +-- +-- One OpenProfile.md per person, generated from what their public profiles +-- say about them. +-- +-- OpenProfile (logicsrc.com/openprofile) is one Markdown file that ties a +-- name to its accounts, its topics and its home page. A person who publishes +-- their own is the authority; for everyone else this table holds the version +-- OutreachGraph assembled from the profile page it was handed, the network's +-- public API, and the OpenGraph tags and rel=me links on the site that profile +-- points at. +-- +-- Keyed by person rather than by an id of its own because there is exactly one +-- current profile per person: a later run replaces it. `sources_json` records +-- every URL that contributed, so a reader can tell a claim from a corroborated +-- fact, and `published_url` is set only when the person serves an +-- OpenProfile.md themselves, in which case that file is what the markdown holds. + +CREATE TABLE openprofiles ( + person_id TEXT PRIMARY KEY REFERENCES people(id) ON DELETE CASCADE, + markdown TEXT NOT NULL, + sources_json TEXT NOT NULL DEFAULT '[]', + published_url TEXT, + generated_at TEXT NOT NULL +); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 31630ca..59cb7cf 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -8,6 +8,17 @@ */ export { runCrawlJob, type CrawlJobDeps, type CrawlJobResult } from './crawl'; +export { + intakeSocialPeople, + normaliseSocialInput, + profileUrlFor, + type IntakeDeps, + type IntakeInput, + type IntakePerson, + type IntakeResult, + type SocialPersonInput, +} from './social-intake'; +export { runOpenProfileJob, type OpenProfileDeps, type OpenProfileResult } from './openprofile'; export { regenerateRecommendations, type RegenerateInput, diff --git a/packages/pipeline/src/jobs.ts b/packages/pipeline/src/jobs.ts index c35925b..4763271 100644 --- a/packages/pipeline/src/jobs.ts +++ b/packages/pipeline/src/jobs.ts @@ -47,6 +47,15 @@ export const JOB_KINDS = [ * runs for people found on the page *this* time. */ 'regenerate_recommendations', + /** + * Build one person's OpenProfile.md from their public profiles. + * + * Queued by the social intake route for every person a client hands over, + * one job per person so a profile page that times out costs one person and + * not the batch. Reads the network's public API and the site the profile + * links to, records what they corroborate, then re-decides the person. + */ + 'openprofile', ] as const; export type JobKind = (typeof JOB_KINDS)[number]; diff --git a/packages/pipeline/src/openprofile.ts b/packages/pipeline/src/openprofile.ts new file mode 100644 index 0000000..48a2038 --- /dev/null +++ b/packages/pipeline/src/openprofile.ts @@ -0,0 +1,404 @@ +/** + * The `openprofile` job: read what a person's public profiles say, write + * their OpenProfile.md, keep what the sources corroborate, and re-decide them. + * + * Order of trust, and why it matters here: the network's own API is the + * person describing themselves; the site their profile links to is a page + * they control; a `rel="me"` from that site back to the profile is the + * person confirming, in two places, that both are theirs. Only that last + * case raises identity confidence past the handle-only floor, because only + * it is two sources agreeing rather than one source repeated. + * + * Nothing here sends anything. The most it does is queue a recommendation, + * which the same approval path as every other card still gates. + */ + +import { newId, type Network } from '@outreachgraph/domain'; +import { queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { + buildOpenProfile, + extractProfilePage, + fetchPage, + mergeFacts, + networkForUrl, + readBlueskyProfile, + readMastodonProfile, + readPublishedOpenProfile, + wellKnownOpenProfile, + type FetchLike, + type ProfileAccount, + type ProfileFacts, +} from '@outreachgraph/providers'; +import { regenerateFor } from './pipeline'; +import type { QueuedJob } from './queue'; + +/** Two sources agreed: the profile names the site and the site names the profile. */ +const CORROBORATED_CONFIDENCE = 0.9; +/** The network answered for the handle with a real profile. One source, but a live one. */ +const PROFILE_SEEN_CONFIDENCE = 0.5; +/** A link the person put in their own profile, unverified by the other end. */ +const LINKED_CONFIDENCE = 0.6; + +export interface OpenProfileDeps { + readonly db: Client; + readonly fetchImpl?: FetchLike; + readonly now?: Date; + /** Skip the recommendation step, for callers that only want the file. */ + readonly regenerate?: boolean; +} + +export interface OpenProfileResult { + readonly personId: string; + readonly outcome: 'ok' | 'published' | 'unreadable' | 'skipped'; + /** Accounts the profile now lists, `me` and `link` together. */ + readonly accounts: number; + /** Identities added to the person from what was read. */ + readonly identities: number; + readonly corroborated: boolean; + readonly recommendationId?: string | undefined; + readonly detail?: string | undefined; +} + +interface IdentityRow { + id: string; + network: Network; + handle: string; + platform_user_id: string | null; + profile_url: string | null; + confidence: number; +} + +function hostOf(url: string): string | undefined { + try { + return new URL(url).hostname.replace(/^www\./, '').toLowerCase(); + } catch { + return undefined; + } +} + +function sameProfile(a: string, b: string): boolean { + const norm = (url: string) => + url + .replace(/^https?:\/\/(www\.)?/i, '') + .replace(/\/+$/, '') + .toLowerCase(); + return norm(a) === norm(b); +} + +/** Read the profile itself, by whichever door the network leaves open. */ +async function readProfile( + identity: IdentityRow, + profileUrl: string | undefined, + deps: OpenProfileDeps, +): Promise { + const options = { ...(deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}) }; + if (identity.network === 'bluesky') return readBlueskyProfile(identity.handle, options); + if (identity.network === 'mastodon') + return readMastodonProfile(profileUrl ?? identity.handle, options); + if (!profileUrl) return undefined; + const page = await fetchPage(profileUrl, options); + if (page.outcome !== 'ok' || !page.html) return undefined; + return extractProfilePage(page.html, page.finalUrl); +} + +/** The person's home page: OpenGraph, rel=me links, and their own OpenProfile.md if any. */ +async function readSite( + web: string, + deps: OpenProfileDeps, +): Promise<{ facts?: ProfileFacts; published?: { url: string; markdown: string } }> { + const options = { ...(deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}) }; + const page = await fetchPage(web, options); + const facts = + page.outcome === 'ok' && page.html ? extractProfilePage(page.html, page.finalUrl) : undefined; + + const candidates = [facts?.openprofileUrl, wellKnownOpenProfile(page.finalUrl ?? web)].filter( + (url): url is string => Boolean(url), + ); + const published = await readPublishedOpenProfile(candidates, options); + return { ...(facts ? { facts } : {}), ...(published ? { published } : {}) }; +} + +/** + * Keep every account the sources named as a social identity, at a confidence + * that says how it was found. Handle-and-network duplicates are skipped; an + * existing row is only ever raised, never lowered. + */ +async function recordIdentities( + db: Client, + personId: string, + accounts: readonly ProfileAccount[], + stamp: string, +): Promise { + const existing = await queryAll<{ + network: string; + handle: string; + confidence: number; + id: string; + }>(db, 'SELECT id, network, handle, confidence FROM social_identities WHERE person_id = ?', [ + personId, + ]); + const known = new Map(existing.map((row) => [`${row.network}:${row.handle.toLowerCase()}`, row])); + let added = 0; + + for (const entry of accounts) { + let network: Network | undefined; + let handle: string | undefined; + if (/^mailto:/i.test(entry.url)) { + network = 'email'; + handle = entry.url + .replace(/^mailto:/i, '') + .split('?')[0] + ?.trim() + .toLowerCase(); + } else { + network = entry.network ?? (entry.relation === 'me' ? 'website' : undefined); + handle = entry.network ? handleOf(entry.url) : hostOf(entry.url); + } + if (!network || !handle) continue; + + const confidence = entry.relation === 'me' ? CORROBORATED_CONFIDENCE : LINKED_CONFIDENCE; + const key = `${network}:${handle.toLowerCase()}`; + const found = known.get(key); + if (found) { + if (confidence > found.confidence) { + await db.execute({ + sql: 'UPDATE social_identities SET confidence = ?, last_verified_at = ? WHERE id = ?', + args: [confidence, stamp, found.id], + }); + } + continue; + } + await db.execute({ + sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, profile_url, + confidence, source_type, verified_by, first_seen_at, last_verified_at) + VALUES (?, ?, ?, ?, NULL, ?, ?, 'public_web', ?, ?, ?)`, + args: [ + newId('socialIdentity'), + personId, + network, + handle, + /^mailto:/i.test(entry.url) ? null : entry.url, + confidence, + JSON.stringify(entry.relation === 'me' ? ['rel=me'] : []), + stamp, + stamp, + ], + }); + known.set(key, { id: '', network, handle, confidence }); + added += 1; + } + return added; +} + +function handleOf(url: string): string | undefined { + try { + const parsed = new URL(url); + if (networkForUrl(url) === 'mastodon') { + const user = parsed.pathname.split('/').filter(Boolean)[0]?.replace(/^@/, ''); + return user ? (user.includes('@') ? user : `${user}@${parsed.hostname}`) : undefined; + } + const segments = parsed.pathname.split('/').filter(Boolean); + const first = segments[0]; + if (!first) return undefined; + const nested = /^(in|profile|user|u|c|channel)$/i.test(first) ? segments[1] : first; + return nested?.replace(/^@/, ''); + } catch { + return undefined; + } +} + +export async function runOpenProfileJob( + deps: OpenProfileDeps, + job: QueuedJob, +): Promise { + const { db } = deps; + const stamp = (deps.now ?? new Date()).toISOString(); + const payload = job.payload as { personId?: string; profileUrl?: string; campaignId?: string }; + const personId = payload.personId; + if (!personId) throw new Error('openprofile needs personId'); + + const person = await queryOne<{ + id: string; + display_name: string; + identity_confidence: number; + avatar_url: string | null; + status: string; + }>( + db, + 'SELECT id, display_name, identity_confidence, avatar_url, status FROM people WHERE id = ?', + [personId], + ); + if (!person || person.status === 'deleted') + return { + personId, + outcome: 'skipped', + accounts: 0, + identities: 0, + corroborated: false, + detail: 'no such person', + }; + + const identities = await queryAll( + db, + `SELECT id, network, handle, platform_user_id, profile_url, confidence FROM social_identities + WHERE person_id = ? ORDER BY confidence DESC, first_seen_at ASC`, + [personId], + ); + const primary = + identities.find( + (row) => + payload.profileUrl && row.profile_url && sameProfile(row.profile_url, payload.profileUrl), + ) ?? + identities.find((row) => row.network !== 'email' && row.network !== 'website') ?? + identities[0]; + if (!primary) + return { + personId, + outcome: 'skipped', + accounts: 0, + identities: 0, + corroborated: false, + detail: 'no social identity', + }; + + const profileUrl = payload.profileUrl ?? primary.profile_url ?? undefined; + const profile = await readProfile(primary, profileUrl, deps); + if (!profile) { + return { + personId, + outcome: 'unreadable', + accounts: 0, + identities: 0, + corroborated: false, + detail: `could not read ${profileUrl ?? primary.handle}`, + }; + } + const sources: string[] = [profile.source]; + const facts: ProfileFacts[] = [profile]; + + // The home page, when the profile names one: its card, and whether it points back. + let corroborated = false; + let published: { url: string; markdown: string } | undefined; + const web = + profile.web ?? + profile.accounts.find((entry) => !entry.network && !/^mailto:/i.test(entry.url))?.url; + if (web) { + const site = await readSite(web, deps); + if (site.facts) { + sources.push(site.facts.source); + const profileHost = hostOf(profile.source); + corroborated = site.facts.accounts.some( + (entry) => + entry.relation === 'me' && + (sameProfile(entry.url, profile.source) || + (profileUrl && sameProfile(entry.url, profileUrl)) || + (entry.network === primary.network && + hostOf(entry.url) === profileHost && + handleOf(entry.url)?.toLowerCase() === primary.handle.toLowerCase())), + ); + // A page that vouches for the profile is one whose links we take as the + // person's; one that does not is still their card, but its other links + // are only links. + facts.push( + corroborated + ? site.facts + : { + ...site.facts, + accounts: site.facts.accounts.map((entry) => ({ + ...entry, + relation: 'link' as const, + })), + }, + ); + } + if (site.published) { + published = site.published; + sources.push(site.published.url); + } + } + + const merged = mergeFacts(primary.handle, profile.source, facts); + const markdown = published?.markdown ?? buildOpenProfile({ ...merged, web: merged.web ?? web }); + + await db.execute({ + sql: `INSERT INTO openprofiles (person_id, markdown, sources_json, published_url, generated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(person_id) DO UPDATE SET markdown = excluded.markdown, sources_json = excluded.sources_json, + published_url = excluded.published_url, generated_at = excluded.generated_at`, + args: [personId, markdown, JSON.stringify(sources), published?.url ?? null, stamp], + }); + + // What the sources corroborate, kept where the rest of the product reads it. + const identitiesAdded = await recordIdentities( + db, + personId, + merged.accounts.concat( + merged.email + ? [ + { + url: `mailto:${merged.email}`, + relation: corroborated ? 'me' : 'link', + label: 'Email', + }, + ] + : [], + ), + stamp, + ); + + const confidence = Math.max( + person.identity_confidence, + corroborated ? CORROBORATED_CONFIDENCE : PROFILE_SEEN_CONFIDENCE, + ); + const displayName = + person.display_name === primary.handle && merged.name && merged.name !== primary.handle + ? merged.name + : person.display_name; + await db.execute({ + sql: `UPDATE people SET display_name = ?, identity_confidence = ?, avatar_url = COALESCE(avatar_url, ?), + avatar_source = CASE WHEN avatar_url IS NULL AND ? IS NOT NULL THEN 'profile' ELSE avatar_source END, + last_resolved_at = ?, updated_at = ? + WHERE id = ?`, + args: [ + displayName, + confidence, + merged.avatar ?? null, + merged.avatar ?? null, + stamp, + stamp, + personId, + ], + }); + await db.execute({ + sql: 'UPDATE social_identities SET confidence = MAX(confidence, ?), last_verified_at = ?, platform_user_id = COALESCE(platform_user_id, ?) WHERE id = ?', + args: [confidence, stamp, profile.platformUserId ?? null, primary.id], + }); + + let recommendationId: string | undefined; + if (deps.regenerate !== false) { + const campaignId = + payload.campaignId ?? + ( + await queryOne<{ campaign_id: string }>( + db, + `SELECT campaign_id FROM campaign_people WHERE person_id = ? AND workspace_id = ? ORDER BY updated_at DESC LIMIT 1`, + [personId, job.workspaceId], + ) + )?.campaign_id; + if (campaignId) { + recommendationId = await regenerateFor( + { db, workspaceId: job.workspaceId, campaignId, providers: [] }, + personId, + ); + } + } + + return { + personId, + outcome: published ? 'published' : 'ok', + accounts: merged.accounts.length, + identities: identitiesAdded, + corroborated, + recommendationId, + }; +} diff --git a/packages/pipeline/src/social-intake.test.ts b/packages/pipeline/src/social-intake.test.ts new file mode 100644 index 0000000..f3b32b7 --- /dev/null +++ b/packages/pipeline/src/social-intake.test.ts @@ -0,0 +1,402 @@ +/** + * People handed over by a social client, and the OpenProfile read that + * follows. + * + * What has to hold: a handle opens one person, once, at the handle-only + * confidence; the bio is a signal so a card can trigger; the same handle sent + * again is the same person with no second signal and no second job; junk is + * rejected by name and never fails the batch. Then the job: it reads the + * network's profile and the linked site, a rel=me back from the site raises + * the person past the floor and keeps the site's accounts as identities, the + * Markdown is stored, and the person is re-decided. + */ + +import { afterEach, describe, expect, test } from 'bun:test'; +import { queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { intakeSocialPeople, normaliseSocialInput } from './social-intake'; +import { runOpenProfileJob } from './openprofile'; +import type { QueuedJob } from './queue'; + +let seeded: SeededDatabase | undefined; + +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +async function db(label: string): Promise { + seeded = await seedDatabase(label); + return seeded.db; +} + +const ADA = { + network: 'bluesky', + handle: '@ada.example', + displayName: 'Ada', + bio: 'Engines and poetry. https://ada.example', + avatarUrl: 'https://cdn.bsky.app/ada.jpg', + via: 'following', +}; + +describe('normaliseSocialInput', () => { + test('drops the @, keeps a Fediverse host, derives the profile URL, refuses junk', () => { + expect(normaliseSocialInput({ network: 'Bluesky', handle: '@ada.example' })).toEqual({ + network: 'bluesky', + handle: 'ada.example', + profileUrl: 'https://bsky.app/profile/ada.example', + }); + expect(normaliseSocialInput({ network: 'mastodon', handle: 'ada@hachyderm.io' })).toEqual({ + network: 'mastodon', + handle: 'ada@hachyderm.io', + profileUrl: 'https://hachyderm.io/@ada', + }); + expect(normaliseSocialInput({ network: 'tiktok', handle: 'ada' })).toEqual({ + reason: 'unknown network tiktok', + }); + expect(normaliseSocialInput({ network: 'x', handle: 'not a handle' })).toEqual({ + reason: 'not a handle', + }); + expect( + normaliseSocialInput({ network: 'x', handle: 'ada', profileUrl: 'javascript:alert(1)' }), + ).toEqual({ + network: 'x', + handle: 'ada', + profileUrl: undefined, + }); + }); +}); + +describe('intakeSocialPeople', () => { + test('opens a person once, in the campaign, with a bio signal and one openprofile job', async () => { + const client = await db('intake-once'); + const input = { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + source: 'myna', + people: [ADA], + }; + + const first = await intakeSocialPeople({ db: client }, input); + expect(first).toMatchObject({ created: 1, existing: 0, queued: 1, rejected: [] }); + const [person] = first.people; + expect(person).toMatchObject({ + network: 'bluesky', + handle: 'ada.example', + created: true, + queued: true, + }); + + const row = await queryOne<{ + display_name: string; + identity_confidence: number; + avatar_url: string; + avatar_source: string; + }>( + client, + 'SELECT display_name, identity_confidence, avatar_url, avatar_source FROM people WHERE id = ?', + [person!.id], + ); + expect(row).toEqual({ + display_name: 'Ada', + identity_confidence: 0.35, + avatar_url: 'https://cdn.bsky.app/ada.jpg', + avatar_source: 'myna', + }); + + const identity = await queryOne<{ handle: string; profile_url: string; confidence: number }>( + client, + 'SELECT handle, profile_url, confidence FROM social_identities WHERE person_id = ? AND network = ?', + [person!.id, 'bluesky'], + ); + expect(identity).toEqual({ + handle: 'ada.example', + profile_url: 'https://bsky.app/profile/ada.example', + confidence: 0.35, + }); + + const membership = await queryOne<{ status: string }>( + client, + 'SELECT status FROM campaign_people WHERE campaign_id = ? AND person_id = ?', + [SEED.campaignId, person!.id], + ); + expect(membership?.status).toBe('discovered'); + + const signals = await queryAll<{ signal_type: string; subtype: string; summary: string }>( + client, + 'SELECT signal_type, subtype, summary FROM signals WHERE person_id = ?', + [person!.id], + ); + expect(signals).toEqual([ + { + signal_type: 'content_topic', + subtype: 'social_bio', + summary: 'ada.example: Engines and poetry. https://ada.example', + }, + ]); + + const jobs = await queryAll<{ kind: string; payload_json: string; dedupe_key: string }>( + client, + 'SELECT kind, payload_json, dedupe_key FROM jobs WHERE workspace_id = ? AND kind = ?', + [SEED.workspaceId, 'openprofile'], + ); + expect(jobs).toHaveLength(1); + expect(jobs[0]!.dedupe_key).toBe(`openprofile:${person!.id}`); + expect(JSON.parse(jobs[0]!.payload_json)).toMatchObject({ + personId: person!.id, + campaignId: SEED.campaignId, + profileUrl: 'https://bsky.app/profile/ada.example', + via: 'following', + }); + + // The same handle again, differently cased: the same person, nothing doubled. + const second = await intakeSocialPeople( + { db: client }, + { ...input, people: [{ ...ADA, handle: 'Ada.Example' }] }, + ); + expect(second).toMatchObject({ created: 0, existing: 1, queued: 0 }); + expect(second.people[0]!.id).toBe(person!.id); + expect( + await queryAll(client, 'SELECT id FROM signals WHERE person_id = ?', [person!.id]), + ).toHaveLength(1); + expect( + await queryAll(client, 'SELECT id FROM jobs WHERE kind = ?', ['openprofile']), + ).toHaveLength(1); + }); + + test('junk is rejected by name and the rest of the batch still lands', async () => { + const client = await db('intake-reject'); + const result = await intakeSocialPeople( + { db: client }, + { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + source: 'myna', + people: [ + { network: 'tiktok', handle: 'x' }, + { network: 'x', handle: '' }, + { network: 'mastodon', handle: 'bob@mastodon.social' }, + ], + }, + ); + expect(result.rejected).toEqual([ + { handle: 'x', reason: 'unknown network tiktok' }, + { handle: '', reason: 'not a handle' }, + ]); + expect(result.created).toBe(1); + expect(result.people[0]).toMatchObject({ network: 'mastodon', handle: 'bob@mastodon.social' }); + }); +}); + +const SITE_HTML = `Ada Lovelace + + + +Blueskymail`; + +function stubFetch(pages: Record Response>): typeof fetch { + return (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input); + const found = pages[url]; + return found ? found() : new Response('', { status: 404 }); + }) as unknown as typeof fetch; +} + +describe('runOpenProfileJob', () => { + test('reads the profile and the site, keeps what they corroborate, writes the file, re-decides', async () => { + const client = await db('openprofile-job'); + const intake = await intakeSocialPeople( + { db: client }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId, source: 'myna', people: [ADA] }, + ); + const personId = intake.people[0]!.id; + const job = await queryOne<{ id: string; payload_json: string }>( + client, + 'SELECT id, payload_json FROM jobs WHERE kind = ?', + ['openprofile'], + ); + + const fetchImpl = stubFetch({ + 'https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=ada.example': () => + new Response( + JSON.stringify({ + did: 'did:plc:ada', + handle: 'ada.example', + displayName: 'Ada Lovelace', + description: 'Engines and poetry. https://ada.example', + avatar: 'https://cdn.bsky.app/ada.jpg', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ), + 'https://ada.example/': () => + new Response(SITE_HTML, { status: 200, headers: { 'content-type': 'text/html' } }), + 'https://ada.example': () => + new Response(SITE_HTML, { status: 200, headers: { 'content-type': 'text/html' } }), + }); + + const queued: QueuedJob = { + id: job!.id, + workspaceId: SEED.workspaceId, + kind: 'openprofile', + payload: JSON.parse(job!.payload_json), + attempts: 0, + maxAttempts: 5, + }; + const result = await runOpenProfileJob({ db: client, fetchImpl }, queued); + expect(result).toMatchObject({ personId, outcome: 'ok', corroborated: true }); + expect(result.identities).toBeGreaterThanOrEqual(2); + + const stored = await queryOne<{ + markdown: string; + sources_json: string; + published_url: string | null; + }>( + client, + 'SELECT markdown, sources_json, published_url FROM openprofiles WHERE person_id = ?', + [personId], + ); + expect(stored?.published_url).toBeNull(); + expect(JSON.parse(stored!.sources_json)).toEqual([ + 'https://bsky.app/profile/ada.example', + 'https://ada.example/', + ]); + expect(stored!.markdown).toContain('# Ada Lovelace'); + expect(stored!.markdown).toContain('- **Web**: https://ada.example'); + expect(stored!.markdown).toContain('- **Email**: ada@example.com'); + expect(stored!.markdown).toContain('- [Bluesky](https://bsky.app/profile/ada.example)'); + expect(stored!.markdown).toContain('- [GitHub](https://github.com/ada)'); + + const person = await queryOne<{ display_name: string; identity_confidence: number }>( + client, + 'SELECT display_name, identity_confidence FROM people WHERE id = ?', + [personId], + ); + expect(person).toEqual({ display_name: 'Ada', identity_confidence: 0.9 }); + + const identities = await queryAll<{ + network: string; + handle: string; + confidence: number; + platform_user_id: string | null; + }>( + client, + 'SELECT network, handle, confidence, platform_user_id FROM social_identities WHERE person_id = ? ORDER BY network, handle', + [personId], + ); + expect(identities).toEqual([ + { + network: 'bluesky', + handle: 'ada.example', + confidence: 0.9, + platform_user_id: 'did:plc:ada', + }, + { network: 'email', handle: 'ada@example.com', confidence: 0.9, platform_user_id: null }, + { network: 'github', handle: 'ada', confidence: 0.9, platform_user_id: null }, + ]); + + // A card exists for them now: the bio signal was the trigger. + const cards = await queryAll<{ action: string; status: string }>( + client, + 'SELECT action, status FROM recommendations WHERE person_id = ?', + [personId], + ); + expect(cards.length).toBeGreaterThanOrEqual(1); + expect(result.recommendationId).toBeDefined(); + }); + + test('a site that does not link back leaves the person at the profile-seen confidence', async () => { + const client = await db('openprofile-uncorroborated'); + const intake = await intakeSocialPeople( + { db: client }, + { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + source: 'myna', + people: [{ network: 'bluesky', handle: 'bob.example', bio: 'See https://bob.example' }], + }, + ); + const personId = intake.people[0]!.id; + const fetchImpl = stubFetch({ + 'https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=bob.example': () => + new Response( + JSON.stringify({ + did: 'did:plc:bob', + handle: 'bob.example', + displayName: 'Bob', + description: 'See https://bob.example', + }), + { status: 200 }, + ), + 'https://bob.example/': () => + new Response( + 'Bobgh', + { status: 200, headers: { 'content-type': 'text/html' } }, + ), + }); + const result = await runOpenProfileJob( + { db: client, fetchImpl, regenerate: false }, + { + id: 'job_x', + workspaceId: SEED.workspaceId, + kind: 'openprofile', + payload: { personId, campaignId: SEED.campaignId }, + attempts: 0, + maxAttempts: 5, + }, + ); + expect(result).toMatchObject({ + outcome: 'ok', + corroborated: false, + recommendationId: undefined, + }); + + const person = await queryOne<{ identity_confidence: number }>( + client, + 'SELECT identity_confidence FROM people WHERE id = ?', + [personId], + ); + expect(person?.identity_confidence).toBe(0.5); + const github = await queryOne<{ confidence: number }>( + client, + 'SELECT confidence FROM social_identities WHERE person_id = ? AND network = ?', + [personId, 'github'], + ); + expect(github?.confidence).toBe(0.6); + const stored = await queryOne<{ markdown: string }>( + client, + 'SELECT markdown FROM openprofiles WHERE person_id = ?', + [personId], + ); + expect(stored!.markdown).toContain('## Links\n\n- [GitHub](https://github.com/someone-else)'); + }); + + test('a profile nobody answers for is reported, not thrown', async () => { + const client = await db('openprofile-unreadable'); + const intake = await intakeSocialPeople( + { db: client }, + { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + source: 'myna', + people: [{ network: 'bluesky', handle: 'gone.example' }], + }, + ); + const result = await runOpenProfileJob( + { db: client, fetchImpl: stubFetch({}) }, + { + id: 'job_y', + workspaceId: SEED.workspaceId, + kind: 'openprofile', + payload: { personId: intake.people[0]!.id }, + attempts: 0, + maxAttempts: 5, + }, + ); + expect(result.outcome).toBe('unreadable'); + expect( + await queryOne(client, 'SELECT person_id FROM openprofiles WHERE person_id = ?', [ + intake.people[0]!.id, + ]), + ).toBeUndefined(); + }); +}); diff --git a/packages/pipeline/src/social-intake.ts b/packages/pipeline/src/social-intake.ts new file mode 100644 index 0000000..16bae45 --- /dev/null +++ b/packages/pipeline/src/social-intake.ts @@ -0,0 +1,357 @@ +/** + * People handed over from a social client. + * + * myna (and anything else that follows people on a network) knows a handle, + * a display name, a bio and a profile URL, and nothing more. That is enough + * to open a person here and put them in a campaign, so the rest of the + * machinery can decide whether they are worth an offer: the bio becomes a + * signal, the profile URL becomes an `openprofile` job that reads what the + * profile and its home page say, and the recommendation engine re-decides + * the person once that has landed. + * + * What this does not do is raise anybody's identity confidence. A handle the + * caller followed is still a handle. The job that reads the profile is what + * finds the rel=me link back and earns the confidence. + */ + +import { isNetwork, newId, type Network } from '@outreachgraph/domain'; +import { queryOne, type Client } from '@outreachgraph/db'; +import { enqueue } from './queue'; +import { recordDiscovered } from './stages'; + +/** Matches `HANDLE_ONLY_CONFIDENCE` in listen.ts: a handle is a claim, not an identity. */ +const HANDLE_ONLY_CONFIDENCE = 0.35; +/** A bio is the person describing themselves; it says topic, not fit. */ +const BIO_SIGNAL_CONFIDENCE = 0.6; +const BIO_SIGNAL_RELEVANCE = 0.4; + +export interface SocialPersonInput { + readonly network: string; + readonly handle: string; + readonly profileUrl?: string | undefined; + readonly platformUserId?: string | undefined; + readonly displayName?: string | undefined; + readonly bio?: string | undefined; + readonly avatarUrl?: string | undefined; + readonly followers?: number | undefined; + /** How the caller came by them: `follow`, `following`, `followers`, `graph`. */ + readonly via?: string | undefined; +} + +export interface IntakeDeps { + readonly db: Client; + readonly now?: Date; +} + +export interface IntakeInput { + readonly workspaceId: string; + readonly campaignId: string; + readonly people: readonly SocialPersonInput[]; + /** The client that sent them, recorded on every signal and provenance row. */ + readonly source: string; +} + +export interface IntakePerson { + readonly id: string; + readonly network: Network; + readonly handle: string; + readonly created: boolean; + /** False when an openprofile job for them was already outstanding. */ + readonly queued: boolean; +} + +export interface IntakeResult { + readonly people: readonly IntakePerson[]; + readonly created: number; + readonly existing: number; + readonly queued: number; + readonly rejected: readonly { handle: string; reason: string }[]; +} + +/** The profile URL a network would serve for a bare handle, when we can say. */ +export function profileUrlFor(network: Network, handle: string): string | undefined { + const clean = handle.replace(/^@/, ''); + switch (network) { + case 'bluesky': + return `https://bsky.app/profile/${clean}`; + case 'x': + return `https://x.com/${clean}`; + case 'github': + return `https://github.com/${clean}`; + case 'reddit': + return `https://www.reddit.com/user/${clean}`; + case 'instagram': + return `https://www.instagram.com/${clean}/`; + case 'mastodon': { + const at = clean.indexOf('@'); + return at > 0 ? `https://${clean.slice(at + 1)}/@${clean.slice(0, at)}` : undefined; + } + default: + return undefined; + } +} + +/** Normalise what a client sends: drop the `@`, keep a Fediverse host, refuse junk. */ +export function normaliseSocialInput( + input: SocialPersonInput, +): { network: Network; handle: string; profileUrl?: string | undefined } | { reason: string } { + const network = String(input.network ?? '') + .trim() + .toLowerCase(); + if (!isNetwork(network)) return { reason: `unknown network ${network || '(empty)'}` }; + + const handle = String(input.handle ?? '') + .trim() + .replace(/^@/, ''); + if (!handle || handle.length > 200 || /[\s<>"']/.test(handle)) return { reason: 'not a handle' }; + + let profileUrl = input.profileUrl?.trim() || profileUrlFor(network, handle); + if (profileUrl) { + try { + const parsed = new URL(profileUrl); + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') profileUrl = undefined; + } catch { + profileUrl = undefined; + } + } + return { network, handle, profileUrl }; +} + +/** + * Open (or find) each person, put them in the campaign, keep their bio as a + * signal, and queue the profile read. Idempotent: the same handle sent twice + * is one person, one membership, one signal, one outstanding job. + */ +export async function intakeSocialPeople( + deps: IntakeDeps, + input: IntakeInput, +): Promise { + const { db } = deps; + const at = deps.now ?? new Date(); + const stamp = at.toISOString(); + const people: IntakePerson[] = []; + const rejected: { handle: string; reason: string }[] = []; + let created = 0; + let existing = 0; + let queued = 0; + + for (const raw of input.people) { + const cleaned = normaliseSocialInput(raw); + if ('reason' in cleaned) { + rejected.push({ handle: String(raw.handle ?? ''), reason: cleaned.reason }); + continue; + } + const { network, handle, profileUrl } = cleaned; + + const found = await findPerson(db, network, handle, raw.platformUserId); + let personId = found; + let isNew = false; + if (!personId) { + personId = newId('person'); + isNew = true; + const avatar = raw.avatarUrl?.trim() || null; + await db.execute({ + sql: `INSERT INTO people (id, display_name, identity_confidence, status, outreach_eligible, + believed_minor, avatar_url, avatar_source, created_at, updated_at) + VALUES (?, ?, ?, 'active', 1, 0, ?, ?, ?, ?)`, + args: [ + personId, + raw.displayName?.trim() || handle, + HANDLE_ONLY_CONFIDENCE, + avatar, + avatar ? input.source : null, + stamp, + stamp, + ], + }); + await db.execute({ + sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, profile_url, + confidence, source_type, verified_by, first_seen_at, last_verified_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'public_web', '[]', ?, ?)`, + args: [ + newId('socialIdentity'), + personId, + network, + handle, + raw.platformUserId?.trim() || null, + profileUrl ?? null, + HANDLE_ONLY_CONFIDENCE, + stamp, + stamp, + ], + }); + await recordProvenance( + db, + personId, + 'display_name', + raw.displayName?.trim() || handle, + input.source, + profileUrl, + stamp, + ); + created += 1; + } else { + existing += 1; + // A person we already had may have arrived without a face or a URL. + if (raw.avatarUrl?.trim()) { + await db.execute({ + sql: `UPDATE people SET avatar_url = ?, avatar_source = ?, updated_at = ? + WHERE id = ? AND avatar_url IS NULL`, + args: [raw.avatarUrl.trim(), input.source, stamp, personId], + }); + } + if (profileUrl) { + await db.execute({ + sql: `UPDATE social_identities SET profile_url = ? WHERE person_id = ? AND network = ? AND profile_url IS NULL`, + args: [profileUrl, personId, network], + }); + } + } + + await db.execute({ + sql: `INSERT OR IGNORE INTO campaign_people (campaign_id, person_id, workspace_id, status, + interaction_state, discovered_at, updated_at) + VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?)`, + args: [input.campaignId, personId, input.workspaceId, stamp, stamp], + }); + if (isNew) { + await recordDiscovered(db, { + workspaceId: input.workspaceId, + campaignId: input.campaignId, + personId, + at: stamp, + }); + } + + const bio = raw.bio?.trim(); + if (bio) + await writeBioSignal(db, { + workspaceId: input.workspaceId, + personId, + network, + handle, + bio, + profileUrl, + stamp, + }); + + const job = await enqueue(db, { + workspaceId: input.workspaceId, + kind: 'openprofile', + payload: { + personId, + campaignId: input.campaignId, + ...(profileUrl ? { profileUrl } : {}), + source: input.source, + via: raw.via ?? null, + }, + dedupeKey: `openprofile:${personId}`, + }); + if (job.queued) queued += 1; + + people.push({ id: personId, network, handle, created: isNew, queued: job.queued }); + } + + return { people, created, existing, queued, rejected }; +} + +async function findPerson( + db: Client, + network: Network, + handle: string, + platformUserId?: string, +): Promise { + if (platformUserId?.trim()) { + const byId = await queryOne<{ person_id: string }>( + db, + `SELECT si.person_id FROM social_identities si JOIN people p ON p.id = si.person_id + WHERE si.network = ? AND si.platform_user_id = ? AND p.status != 'deleted' + ORDER BY si.confidence DESC LIMIT 1`, + [network, platformUserId.trim()], + ); + if (byId) return byId.person_id; + } + const byHandle = await queryOne<{ person_id: string }>( + db, + `SELECT si.person_id FROM social_identities si JOIN people p ON p.id = si.person_id + WHERE si.network = ? AND si.handle = ? COLLATE NOCASE AND p.status != 'deleted' + ORDER BY si.confidence DESC LIMIT 1`, + [network, handle], + ); + return byHandle?.person_id; +} + +async function recordProvenance( + db: Client, + personId: string, + field: string, + value: string, + provider: string, + sourceUrl: string | undefined, + stamp: string, +): Promise { + await db.execute({ + sql: `INSERT INTO field_provenance (id, entity_kind, entity_id, field, value, source_type, provider, + source_record_id, license_class, confidence, observed_at, created_at) + VALUES (?, 'person', ?, ?, ?, 'public_web', ?, ?, 'public', ?, ?, ?)`, + args: [ + newId('fieldProvenance'), + personId, + field, + value, + provider, + sourceUrl ?? null, + HANDLE_ONLY_CONFIDENCE, + stamp, + stamp, + ], + }); +} + +/** + * The bio as a `content_topic` signal, so the recommendation engine has a + * trigger to weigh. One per network: the same bio sent again is the same + * signal, not a fresher one. + */ +async function writeBioSignal( + db: Client, + input: { + workspaceId: string; + personId: string; + network: Network; + handle: string; + bio: string; + profileUrl?: string | undefined; + stamp: string; + }, +): Promise { + const sourceUrl = input.profileUrl ?? `${input.network}:${input.handle}`; + // One bio per network per person. A handle sent again in another case, or + // with a slightly different profile URL, is the same person saying the same + // thing, not a fresher signal. + const already = await queryOne<{ id: string }>( + db, + `SELECT id FROM signals WHERE workspace_id = ? AND person_id = ? AND subtype = 'social_bio' AND network = ? LIMIT 1`, + [input.workspaceId, input.personId, input.network], + ); + if (already) return; + + await db.execute({ + sql: `INSERT INTO signals (id, workspace_id, person_id, network, signal_type, subtype, summary, evidence, + source_url, source_timestamp, observed_at, confidence, relevance, sentiment) + VALUES (?, ?, ?, ?, 'content_topic', 'social_bio', ?, ?, ?, ?, ?, ?, ?, 'neutral')`, + args: [ + newId('signal'), + input.workspaceId, + input.personId, + input.network, + `${input.handle}: ${input.bio.slice(0, 180)}`, + input.bio.slice(0, 2000), + sourceUrl, + input.stamp, + input.stamp, + BIO_SIGNAL_CONFIDENCE, + BIO_SIGNAL_RELEVANCE, + ], + }); +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index a668b71..9d8a5fd 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -92,6 +92,22 @@ export { type SiteExtraction, } from './site/extract'; +export { + buildOpenProfile, + extractProfilePage, + hashtagsIn, + labelFor, + mergeFacts, + readBlueskyProfile, + readMastodonProfile, + readPublishedOpenProfile, + urlsInText, + wellKnownOpenProfile, + type ProfileAccount, + type ProfileFacts, + type ProfileInput, +} from './site/openprofile'; + export { extractWithModel, visibleText, diff --git a/packages/providers/src/site/extract.ts b/packages/providers/src/site/extract.ts index e49e512..8676570 100644 --- a/packages/providers/src/site/extract.ts +++ b/packages/providers/src/site/extract.ts @@ -115,7 +115,7 @@ export function handleFromUrl(url: string): string | undefined { } } -function decodeEntities(text: string): string { +export function decodeEntities(text: string): string { return text .replace(/</g, '<') .replace(/>/g, '>') @@ -126,7 +126,7 @@ function decodeEntities(text: string): string { .replace(/&/g, '&'); } -function collapse(text: string): string { +export function collapse(text: string): string { return decodeEntities(text).replace(/\s+/g, ' ').trim(); } @@ -185,7 +185,11 @@ function urlList(node: unknown, field: string): string[] { return []; } -function metaContent(html: string, attr: 'property' | 'name', key: string): string | undefined { +export function metaContent( + html: string, + attr: 'property' | 'name', + key: string, +): string | undefined { const pattern = new RegExp( `]*${attr}\\s*=\\s*["']${key}["'][^>]*content\\s*=\\s*["']([^"']*)["']`, 'i', diff --git a/packages/providers/src/site/openprofile.test.ts b/packages/providers/src/site/openprofile.test.ts new file mode 100644 index 0000000..162bd0c --- /dev/null +++ b/packages/providers/src/site/openprofile.test.ts @@ -0,0 +1,276 @@ +/** + * OpenProfile assembly from pages and profiles, with no network. + * + * What has to hold: a page's OpenGraph card and rel=me links are read in + * either attribute order; a plain link only counts when it names a network + * we know; the Bluesky and Mastodon readers turn a public profile into the + * same facts a page would; merging lets the most trusted source win and + * never downgrades a `me` to a `link`; and the Markdown that comes out obeys + * the spec's shape, one heading, an identity block, one headline line. + */ + +import { describe, expect, test } from 'bun:test'; +import { + buildOpenProfile, + extractProfilePage, + hashtagsIn, + mergeFacts, + readBlueskyProfile, + readMastodonProfile, + readPublishedOpenProfile, + urlsInText, + wellKnownOpenProfile, +} from './openprofile'; + +const SITE = ` +Ada Lovelace + + + + + + +Bluesky +Mastodon +videos +a friend +mail me +top +`; + +describe('extractProfilePage', () => { + test('reads the card, every rel=me, the mailto and the openprofile link', () => { + const facts = extractProfilePage(SITE, 'https://ada.example/'); + + expect(facts.name).toBe('Ada Lovelace'); + expect(facts.headline).toBe('Writes about machines that do not exist yet. #babbage'); + expect(facts.avatar).toBe('https://ada.example/ada.png'); + expect(facts.openprofileUrl).toBe('https://ada.example/profile.md'); + expect(facts.topics).toEqual(['babbage']); + + const byUrl = Object.fromEntries(facts.accounts.map((entry) => [entry.url, entry])); + expect(byUrl['https://github.com/ada']).toMatchObject({ + relation: 'me', + network: 'github', + label: 'GitHub', + }); + expect(byUrl['https://bsky.app/profile/ada.example']).toMatchObject({ + relation: 'me', + network: 'bluesky', + }); + expect(byUrl['https://mathstodon.xyz/@ada']).toMatchObject({ + relation: 'me', + network: 'mastodon', + }); + expect(byUrl['https://www.youtube.com/@ada']).toMatchObject({ + relation: 'link', + network: 'youtube', + }); + expect(byUrl['mailto:ada@example.com']).toMatchObject({ relation: 'me', label: 'Email' }); + // A plain link to a site we cannot place is not an account. + expect(byUrl['https://example.org/some/other/page']).toBeUndefined(); + }); + + test('a page with no card still yields its title and nothing invented', () => { + const facts = extractProfilePage( + 'Plainhi', + 'https://p.example/', + ); + expect(facts).toMatchObject({ name: 'Plain', accounts: [], topics: [] }); + expect(facts.headline).toBeUndefined(); + expect(facts.avatar).toBeUndefined(); + }); +}); + +describe('text helpers', () => { + test('urls and hashtags come out of a bio', () => { + expect(urlsInText('site: https://ada.example/ and https://github.com/ada.')).toEqual([ + 'https://ada.example/', + 'https://github.com/ada', + ]); + expect(hashtagsIn('I write about #Babbage and #babbage, plus #ai-safety')).toEqual([ + 'babbage', + 'ai-safety', + ]); + }); + + test('the well-known path is derived from any page on the site', () => { + expect(wellKnownOpenProfile('https://ada.example/blog/post?x=1')).toBe( + 'https://ada.example/.well-known/openprofile.md', + ); + expect(wellKnownOpenProfile('not a url')).toBeUndefined(); + }); +}); + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +describe('network readers', () => { + test('a Bluesky actor becomes facts, with the bio links and the DID', async () => { + const calls: string[] = []; + const facts = await readBlueskyProfile('ada.example', { + fetchImpl: async (input) => { + calls.push(String(input)); + return jsonResponse({ + did: 'did:plc:ada', + handle: 'ada.example', + displayName: 'Ada', + description: + 'Engines and poetry.\nMore at https://ada.example and https://github.com/ada #babbage', + avatar: 'https://cdn.bsky.app/ada.jpg', + }); + }, + }); + expect(calls[0]).toBe( + 'https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=ada.example', + ); + expect(facts).toMatchObject({ + source: 'https://bsky.app/profile/ada.example', + name: 'Ada', + headline: 'Engines and poetry.', + web: 'https://ada.example', + platformUserId: 'did:plc:ada', + topics: ['babbage'], + }); + expect(facts?.accounts.map((entry) => entry.url)).toEqual([ + 'https://ada.example', + 'https://github.com/ada', + ]); + }); + + test('a Mastodon account becomes facts, and a verified field is rel=me', async () => { + const facts = await readMastodonProfile('https://mathstodon.xyz/@ada', { + fetchImpl: async (input) => { + expect(String(input)).toBe('https://mathstodon.xyz/api/v1/accounts/lookup?acct=ada'); + return jsonResponse({ + id: '42', + acct: 'ada', + url: 'https://mathstodon.xyz/@ada', + display_name: 'Ada Lovelace', + note: '

Analytical engines. Poetry on Sundays.

', + avatar: 'https://files.mathstodon.xyz/ada.png', + fields: [ + { + name: 'Web', + value: 'ada.example', + verified_at: '2026-01-01T00:00:00Z', + }, + { + name: 'Code', + value: 'github.com/ada', + verified_at: null, + }, + ], + }); + }, + }); + expect(facts).toMatchObject({ + name: 'Ada Lovelace', + headline: 'Analytical engines.', + web: 'https://ada.example', + platformUserId: 'mathstodon.xyz:42', + }); + expect(facts?.accounts).toEqual([ + { url: 'https://ada.example', network: undefined, relation: 'me', label: 'ada.example' }, + { url: 'https://github.com/ada', network: 'github', relation: 'link', label: 'GitHub' }, + ]); + }); + + test('a network that answers 404 yields nothing rather than a half profile', async () => { + expect( + await readBlueskyProfile('nobody.example', { + fetchImpl: async () => new Response('', { status: 404 }), + }), + ).toBeUndefined(); + }); + + test('a published OpenProfile.md is taken only when it is Markdown', async () => { + const pages: Record = { + 'https://ada.example/profile.md': new Response('home', { status: 200 }), + 'https://ada.example/.well-known/openprofile.md': new Response( + '# Ada Lovelace\n\n- **Kind**: person\n', + { status: 200 }, + ), + }; + const found = await readPublishedOpenProfile( + ['https://ada.example/profile.md', 'https://ada.example/.well-known/openprofile.md'], + { + fetchImpl: async (input) => pages[String(input)] ?? new Response('', { status: 404 }), + }, + ); + expect(found?.url).toBe('https://ada.example/.well-known/openprofile.md'); + expect(found?.markdown.startsWith('# Ada Lovelace')).toBe(true); + }); +}); + +describe('mergeFacts and buildOpenProfile', () => { + test('the first source wins on scalars, me beats link, and the profile itself is always listed', () => { + const merged = mergeFacts('ada.example', 'https://bsky.app/profile/ada.example', [ + { + source: 'bsky', + name: 'Ada', + accounts: [ + { url: 'https://github.com/ada', network: 'github', relation: 'link', label: 'GitHub' }, + ], + topics: ['babbage'], + }, + { + source: 'site', + name: 'Ada Lovelace', + headline: 'Writes about machines that do not exist yet.', + avatar: 'https://ada.example/ada.png', + accounts: [ + { url: 'https://github.com/ada', network: 'github', relation: 'me', label: 'GitHub' }, + { url: 'mailto:ada@example.com', relation: 'me', label: 'Email' }, + ], + topics: ['poetry', 'babbage'], + }, + ]); + expect(merged.name).toBe('Ada'); + expect(merged.headline).toBe('Writes about machines that do not exist yet.'); + expect(merged.email).toBe('ada@example.com'); + expect(merged.topics).toEqual(['babbage', 'poetry']); + expect(merged.accounts.map((entry) => [entry.url, entry.relation])).toEqual([ + ['https://bsky.app/profile/ada.example', 'me'], + ['https://github.com/ada', 'me'], + ]); + + const markdown = buildOpenProfile({ ...merged, web: 'https://ada.example' }); + expect(markdown).toBe( + [ + '# Ada', + '', + '- **Kind**: person', + '- **Handle**: @ada.example', + '- **Web**: https://ada.example', + '- **Email**: ada@example.com', + '- **Avatar**: https://ada.example/ada.png', + '', + 'Writes about machines that do not exist yet.', + '', + '## Accounts', + '', + '- [Bluesky](https://bsky.app/profile/ada.example)', + '- [GitHub](https://github.com/ada)', + '', + '## Topics', + '', + '- babbage, poetry', + '', + ].join('\n'), + ); + // One heading, and it is the name. + expect(markdown.match(/^# /gm)).toHaveLength(1); + }); + + test('a person with nothing but a handle still gets a valid file', () => { + const markdown = buildOpenProfile(mergeFacts('@bob', 'https://x.com/bob', [])); + expect(markdown).toBe( + '# bob\n\n- **Kind**: person\n- **Handle**: @bob\n\n## Accounts\n\n- [X](https://x.com/bob)\n', + ); + }); +}); diff --git a/packages/providers/src/site/openprofile.ts b/packages/providers/src/site/openprofile.ts new file mode 100644 index 0000000..fedd8c8 --- /dev/null +++ b/packages/providers/src/site/openprofile.ts @@ -0,0 +1,433 @@ +/** + * OpenProfile.md, assembled from a person's public profiles. + * + * OpenProfile (https://logicsrc.com/openprofile) is one Markdown file that + * says who somebody is and where they are: a name, an identity block, one + * headline, and an Accounts section whose bullets are the URLs that are them. + * A person who serves their own at `/.well-known/openprofile.md` is the + * authority and we keep their file. For everyone else this module builds one + * from three places, in order of how much each is allowed to say: + * + * 1. The network's public API for the profile we were handed (Bluesky, + * Mastodon). Display name, bio, avatar, and the links the person put in + * their own profile. + * 2. The site those links point at: its OpenGraph tags, and every `rel="me"` + * link on it. A `rel="me"` back to the profile is the person confirming, + * on a page they control, that the profile is theirs. + * 3. The profile page's own OpenGraph tags, for networks with no public API. + * + * Everything here is pure over strings and JSON except the two readers that + * take a `fetchImpl`, so a test can hand in pages and never touch the network. + */ + +import { anchors, collapse, decodeEntities, isRelMe, metaContent, networkForUrl } from './extract'; +import { parseFediverseHandle, parseFediverseUrl } from './fediverse'; +import type { FetchLike } from './fetch'; +import { USER_AGENT } from './fetch'; +import type { Network } from '@outreachgraph/domain'; + +/** One account the profile names, with how sure we are it is the same person. */ +export interface ProfileAccount { + readonly url: string; + /** Which network the URL belongs to, when it is one we know. */ + readonly network?: Network | undefined; + /** `me` when the page marked it rel=me or the network verified it; `link` otherwise. */ + readonly relation: 'me' | 'link'; + /** What to call it: "Bluesky", "GitHub", or the site's host for a plain link. */ + readonly label: string; +} + +/** What one source said about the person. */ +export interface ProfileFacts { + readonly source: string; + readonly name?: string | undefined; + readonly headline?: string | undefined; + readonly avatar?: string | undefined; + /** The home page the profile names, when it names one. */ + readonly web?: string | undefined; + readonly accounts: readonly ProfileAccount[]; + /** `#tags` and comma topics the bio carried, lower-cased, deduplicated. */ + readonly topics: readonly string[]; + /** A network-native stable id, such as a Bluesky DID. */ + readonly platformUserId?: string | undefined; + /** Set when the page linked or served an OpenProfile.md of its own. */ + readonly openprofileUrl?: string | undefined; +} + +/** What the builder needs, after every source has been merged. */ +export interface ProfileInput { + readonly name: string; + readonly handle: string; + readonly kind?: 'person' | 'agent' | 'organization'; + readonly headline?: string | undefined; + readonly web?: string | undefined; + readonly avatar?: string | undefined; + readonly email?: string | undefined; + readonly accounts: readonly ProfileAccount[]; + readonly topics: readonly string[]; +} + +const LABELS: Readonly> = { + bluesky: 'Bluesky', + mastodon: 'Mastodon', + x: 'X', + github: 'GitHub', + linkedin: 'LinkedIn', + reddit: 'Reddit', + youtube: 'YouTube', + instagram: 'Instagram', + nostr: 'Nostr', + website: 'Website', +}; + +function hostOf(url: string): string | undefined { + try { + return new URL(url).hostname.replace(/^www\./, ''); + } catch { + return undefined; + } +} + +/** A label for an account: the network's name, or the host for a plain site. */ +export function labelFor(url: string, network?: Network): string { + if (network && LABELS[network]) return LABELS[network] as string; + return hostOf(url) ?? 'Link'; +} + +function stripTags(html: string): string { + return collapse(html.replace(//gi, ' ').replace(/<[^>]+>/g, ' ')); +} + +/** Every http(s) URL written out in plain text. */ +export function urlsInText(text: string): string[] { + const found = new Set(); + for (const match of text.matchAll(/https?:\/\/[^\s<>()"']+/gi)) { + found.add(match[0].replace(/[.,;:!?)]+$/, '')); + } + return [...found]; +} + +/** `#tag` words in a bio, lower-cased and deduplicated. */ +export function hashtagsIn(text: string): string[] { + const found = new Set(); + for (const match of text.matchAll(/(?:^|\s)#([\p{L}\p{N}_-]{2,40})/gu)) { + found.add(match[1]!.toLowerCase()); + } + return [...found]; +} + +function account(url: string, relation: 'me' | 'link'): ProfileAccount { + const network = networkForUrl(url); + return { url, network, relation, label: labelFor(url, network) }; +} + +function samePage(a: string, b: string): boolean { + const norm = (url: string) => + url + .replace(/^https?:\/\/(www\.)?/i, '') + .replace(/\/+$/, '') + .toLowerCase(); + return norm(a) === norm(b); +} + +/** + * What a page says about the person behind it: OpenGraph card, every + * `rel="me"` link, and a linked OpenProfile.md if it advertises one. + * + * The OpenGraph title is the page's name for itself, which on a profile page + * is usually the person and on a home page is usually the site. Callers + * decide which; this only reads. + */ +export function extractProfilePage(html: string, pageUrl: string): ProfileFacts { + const title = + metaContent(html, 'property', 'og:title') ?? + collapse(html.match(/]*>([^<]*)<\/title>/i)?.[1] ?? ''); + const description = + metaContent(html, 'property', 'og:description') ?? metaContent(html, 'name', 'description'); + const image = metaContent(html, 'property', 'og:image'); + + const accounts = new Map(); + let openprofileUrl: string | undefined; + + for (const anchor of anchors(html)) { + const href = resolveHref(anchor.href, pageUrl); + if (!href) continue; + if (/^mailto:/i.test(href)) { + if (isRelMe(anchor.tag)) + accounts.set(href.toLowerCase(), { url: href, relation: 'me', label: 'Email' }); + continue; + } + if (!/^https?:\/\//i.test(href) || samePage(href, pageUrl)) continue; + const relation: 'me' | 'link' = isRelMe(anchor.tag) ? 'me' : 'link'; + const existing = accounts.get(href); + if (existing?.relation === 'me') continue; + // A plain link is only worth keeping when it names a network we know; + // rel=me is kept whatever it points at, because the page said so. + if (relation === 'link' && !networkForUrl(href)) continue; + accounts.set(href, { ...account(href, relation) }); + } + + // `` in the head counts the same as an anchor. + for (const match of html.matchAll(/]*>/gi)) { + const tag = match[0]; + const href = resolveHref(/href\s*=\s*["']([^"']+)["']/i.exec(tag)?.[1], pageUrl); + if (!href) continue; + if (/rel\s*=\s*["'][^"']*\bopenprofile\b/i.test(tag)) openprofileUrl ??= href; + else if (isRelMe(tag) && /^https?:\/\//i.test(href) && !samePage(href, pageUrl)) { + accounts.set(href, { ...account(href, 'me') }); + } + } + + const text = stripTags(html.replace(/|/gi, ' ')); + + return { + source: pageUrl, + name: title || undefined, + headline: description ? collapse(description) : undefined, + avatar: image ? resolveHref(image, pageUrl) : undefined, + accounts: [...accounts.values()], + topics: hashtagsIn(description ?? '') + .concat(hashtagsIn(text.slice(0, 2000))) + .filter((topic, index, all) => all.indexOf(topic) === index), + openprofileUrl, + }; +} + +function resolveHref(href: string | undefined, base: string): string | undefined { + if (!href) return undefined; + const trimmed = decodeEntities(href.trim()); + if (!trimmed || trimmed.startsWith('#') || /^javascript:/i.test(trimmed)) return undefined; + if (/^mailto:/i.test(trimmed)) return trimmed; + try { + return new URL(trimmed, base).toString(); + } catch { + return undefined; + } +} + +interface ReaderOptions { + readonly fetchImpl?: FetchLike; + readonly timeoutMs?: number; +} + +async function getJson(url: string, options: ReaderOptions): Promise { + const call = options.fetchImpl ?? fetch; + try { + const response = await call(url, { + headers: { 'user-agent': USER_AGENT, accept: 'application/json' }, + signal: AbortSignal.timeout(options.timeoutMs ?? 10_000), + }); + if (!response.ok) return undefined; + return (await response.json()) as T; + } catch { + return undefined; + } +} + +interface BlueskyActor { + did?: string; + handle?: string; + displayName?: string; + description?: string; + avatar?: string; +} + +/** A Bluesky profile from the public AppView. No token, no session. */ +export async function readBlueskyProfile( + handle: string, + options: ReaderOptions = {}, +): Promise { + const actor = handle + .replace(/^@/, '') + .replace(/^https?:\/\/bsky\.app\/profile\//i, '') + .replace(/\/.*$/, ''); + if (!actor) return undefined; + const found = await getJson( + `https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=${encodeURIComponent(actor)}`, + options, + ); + if (!found?.handle) return undefined; + + const bio = found.description ?? ''; + const links = urlsInText(bio); + const web = links.find((url) => !networkForUrl(url)); + return { + source: `https://bsky.app/profile/${found.handle}`, + name: found.displayName?.trim() || undefined, + headline: bio + .split(/\r?\n/) + .map((line) => line.trim()) + .find(Boolean), + avatar: found.avatar, + web, + accounts: links.map((url) => account(url, 'link')), + topics: hashtagsIn(bio), + platformUserId: found.did, + }; +} + +interface MastodonAccount { + id?: string; + acct?: string; + url?: string; + display_name?: string; + note?: string; + avatar?: string; + fields?: { name?: string; value?: string; verified_at?: string | null }[]; +} + +/** A Mastodon (or compatible) profile from the instance's public lookup. */ +export async function readMastodonProfile( + ref: string, + options: ReaderOptions = {}, +): Promise { + const parsed = /^https?:\/\//i.test(ref) + ? parseFediverseUrl(ref) + : parseFediverseHandle(ref.startsWith('@') ? ref : `@${ref}`); + if (!parsed) return undefined; + const found = await getJson( + `https://${parsed.host}/api/v1/accounts/lookup?acct=${encodeURIComponent(parsed.user)}`, + options, + ); + if (!found?.acct) return undefined; + + const bio = stripTags(found.note ?? ''); + const accounts = new Map(); + let web: string | undefined; + for (const field of found.fields ?? []) { + for (const url of urlsInText(stripTags(field.value ?? '')).concat( + urlsInText(field.value ?? ''), + )) { + const relation: 'me' | 'link' = field.verified_at ? 'me' : 'link'; + const existing = accounts.get(url); + if (existing?.relation === 'me') continue; + accounts.set(url, account(url, relation)); + if (!web && !networkForUrl(url)) web = url; + } + } + for (const url of urlsInText(bio)) { + if (!accounts.has(url)) accounts.set(url, account(url, 'link')); + if (!web && !networkForUrl(url)) web = url; + } + + return { + source: found.url ?? parsed.profileUrl, + name: found.display_name?.trim() || undefined, + headline: bio.split(/(?<=[.!?])\s+/).find(Boolean), + avatar: found.avatar, + web, + accounts: [...accounts.values()], + topics: hashtagsIn(bio), + platformUserId: found.id ? `${parsed.host}:${found.id}` : undefined, + }; +} + +/** + * The person's own OpenProfile.md at a site, if they serve one. + * + * Tries the linked URL first, then the well-known path. Accepts only a body + * that starts with a Markdown heading, because a site that answers every + * path with its home page would otherwise hand us HTML as a profile. + */ +export async function readPublishedOpenProfile( + candidates: readonly string[], + options: ReaderOptions = {}, +): Promise<{ url: string; markdown: string } | undefined> { + const call = options.fetchImpl ?? fetch; + for (const url of candidates) { + try { + const response = await call(url, { + headers: { 'user-agent': USER_AGENT, accept: 'text/markdown, text/plain;q=0.9, */*;q=0.1' }, + signal: AbortSignal.timeout(options.timeoutMs ?? 10_000), + }); + if (!response.ok) continue; + const body = (await response.text()).trim(); + if (/^#\s+\S/.test(body) && !/^\s*(key: K): ProfileFacts[K] | undefined => + facts.map((fact) => fact[key]).find((value) => value !== undefined && value !== ''); + + const accounts = new Map(); + accounts.set(profileUrl, account(profileUrl, 'me')); + for (const fact of facts) { + for (const entry of fact.accounts) { + const existing = accounts.get(entry.url); + if (existing?.relation === 'me') continue; + accounts.set(entry.url, entry); + } + } + const email = [...accounts.values()] + .find((entry) => /^mailto:/i.test(entry.url)) + ?.url.replace(/^mailto:/i, '') + .split('?')[0]; + + const topics = [...new Set(facts.flatMap((fact) => fact.topics))]; + return { + name: (first('name') as string | undefined) ?? handle.replace(/^@/, ''), + handle: handle.replace(/^@/, ''), + kind: 'person', + headline: first('headline') as string | undefined, + web: first('web') as string | undefined, + avatar: first('avatar') as string | undefined, + email, + accounts: [...accounts.values()].filter((entry) => !/^mailto:/i.test(entry.url)), + topics, + }; +} + +/** Render the Markdown the spec describes. One `#`, an identity block, one line, sections. */ +export function buildOpenProfile(input: ProfileInput): string { + const lines: string[] = [`# ${input.name.trim() || input.handle}`, '']; + lines.push(`- **Kind**: ${input.kind ?? 'person'}`); + lines.push(`- **Handle**: @${input.handle.replace(/^@/, '')}`); + if (input.web) lines.push(`- **Web**: ${input.web}`); + if (input.email) lines.push(`- **Email**: ${input.email}`); + if (input.avatar) lines.push(`- **Avatar**: ${input.avatar}`); + lines.push(''); + if (input.headline) lines.push(input.headline.trim(), ''); + + // The home page is the Web line; listing it again under Links says nothing new. + const isWeb = (entry: ProfileAccount) => Boolean(input.web && samePage(entry.url, input.web)); + const me = input.accounts.filter((entry) => entry.relation === 'me' && !isWeb(entry)); + const links = input.accounts.filter((entry) => entry.relation === 'link' && !isWeb(entry)); + if (me.length) { + lines.push('## Accounts', ''); + for (const entry of me) lines.push(`- [${entry.label}](${entry.url})`); + lines.push(''); + } + if (input.topics.length) { + lines.push('## Topics', '', `- ${input.topics.join(', ')}`, ''); + } + if (links.length) { + lines.push('## Links', ''); + for (const entry of links) lines.push(`- [${entry.label}](${entry.url})`); + lines.push(''); + } + return `${lines.join('\n').trimEnd()}\n`; +}