From e8ab41fc425dc0002ab959d495c7b59cf49d3c53 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 6 Aug 2026 10:58:29 +0200 Subject: [PATCH 01/65] chore: prepare server 1.4.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 6d84d772..99db84dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.3.1 +appVersion=1.4.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From 022d15968b70367103bc0e83652b19852d0c638c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 7 Aug 2026 22:28:37 +0200 Subject: [PATCH 02/65] fix: verify remote login capability --- openapi/components/instance.yaml | 2 +- .../YoutubeRemoteLoginReadinessService.kt | 20 +++++++++-- .../YoutubeRemoteLoginReadinessServiceTest.kt | 35 ++++++++++++++++--- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 676c77c6..385bb0fc 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -44,7 +44,7 @@ InstanceResponse: description: True only when the admin setting is enabled and remote login is ready. youtubeRemoteLoginReady: type: boolean - description: Non-secret readiness state for YouTube remote login. + description: True when Token accepts the shared secret and callback and its browser runtime is available. youtubeRemoteLoginUnavailableReason: type: string nullable: true diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt index 1e3ed387..5ee9abb3 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt @@ -3,8 +3,13 @@ package dev.typetype.server.services import dev.typetype.server.models.YoutubeRemoteLoginStatus import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import java.util.concurrent.TimeUnit class YoutubeRemoteLoginReadinessService( @@ -13,6 +18,7 @@ class YoutubeRemoteLoginReadinessService( private val client: OkHttpClient = defaultClient(), private val nowMs: () -> Long = System::currentTimeMillis, ) { + private val json = Json { encodeDefaults = true } private var cachedStatus: YoutubeRemoteLoginStatus? = null private var cachedUntilMs: Long = 0 private val lock = Any() @@ -40,9 +46,14 @@ class YoutubeRemoteLoginReadinessService( } private fun probeToken(): YoutubeRemoteLoginStatus { + val internalToken = config.internalToken ?: return YoutubeRemoteLoginStatus.NotConfigured val request = Request.Builder() - .url("${config.serviceUrl.trimEnd('/')}/health") - .get() + .url("${config.serviceUrl.trimEnd('/')}/youtube-remote-login/readiness") + .header(INTERNAL_HEADER, internalToken) + .post( + json.encodeToString(YoutubeRemoteLoginReadinessRequest(config.callbackUrl)) + .toRequestBody(JSON_MEDIA_TYPE) + ) .build() return runCatching { client.newCall(request).execute().use { @@ -53,6 +64,8 @@ class YoutubeRemoteLoginReadinessService( private companion object { const val CACHE_TTL_MS = 30_000L + const val INTERNAL_HEADER = "X-Internal-Token" + val JSON_MEDIA_TYPE = "application/json".toMediaType() fun defaultClient(): OkHttpClient = OkHttpClient.Builder() @@ -62,3 +75,6 @@ class YoutubeRemoteLoginReadinessService( .build() } } + +@Serializable +private data class YoutubeRemoteLoginReadinessRequest(val callbackUrl: String) diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt index 7126a130..c66879f5 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt @@ -11,6 +11,7 @@ import okhttp3.OkHttpClient import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody +import okio.Buffer import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -34,17 +35,22 @@ class YoutubeRemoteLoginReadinessServiceTest { } @Test - fun `token health failure returns token unreachable`() = runBlocking { - val service = service(config("secret"), sessionConfigured = true, client = client(500)) + fun `token without capability endpoint returns token unreachable`() = runBlocking { + val service = service(config("secret"), sessionConfigured = true, client = client(404)) assertEquals(YoutubeRemoteLoginStatus.TokenUnreachable, service.status(adminEnabled = true)) } @Test - fun `token health success returns ready`() = runBlocking { - val service = service(config("secret"), sessionConfigured = true, client = client(200)) + fun `token capability success returns ready`() = runBlocking { + val recorder = RequestRecorder() + val service = service(config("secret"), sessionConfigured = true, client = client(204, recorder = recorder)) assertEquals(YoutubeRemoteLoginStatus.Ready, service.status(adminEnabled = true)) + assertEquals("POST", recorder.method) + assertEquals("/youtube-remote-login/readiness", recorder.path) + assertEquals("secret", recorder.internalToken) + assertEquals("{\"callbackUrl\":\"http://server/internal/youtube-remote-login/callback\"}", recorder.body) } private fun service( @@ -61,9 +67,14 @@ class YoutubeRemoteLoginReadinessServiceTest { private fun config(internalToken: String?): YoutubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig("http://token", "http://server", internalToken, 480_000, 2, 524_288, 4096, 2) - private fun client(code: Int, calls: Counter = Counter()): OkHttpClient = + private fun client( + code: Int, + calls: Counter = Counter(), + recorder: RequestRecorder? = null, + ): OkHttpClient = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> calls.value += 1 + recorder?.record(chain.request()) Response.Builder() .request(chain.request()) .protocol(Protocol.HTTP_1_1) @@ -76,4 +87,18 @@ class YoutubeRemoteLoginReadinessServiceTest { private class Counter { var value: Int = 0 } + + private class RequestRecorder { + var method: String? = null + var path: String? = null + var internalToken: String? = null + var body: String? = null + + fun record(request: okhttp3.Request) { + method = request.method + path = request.url.encodedPath + internalToken = request.header("X-Internal-Token") + body = Buffer().also { request.body?.writeTo(it) }.readUtf8() + } + } } From c8558cc7eabfa197226f8685d034e52c2f86f320 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:03 +0200 Subject: [PATCH 03/65] fix: restrict public proxy destinations --- openapi/paths/proxy.yaml | 6 +- .../server/services/BilibiliRangeProxy.kt | 8 +- .../server/services/HlsManifestService.kt | 5 +- .../server/services/NicoVideoProxyService.kt | 12 +- .../server/services/OkHttpProxyService.kt | 9 +- .../server/services/ProxyHttpExecutor.kt | 89 ++++++++++++ .../typetype/server/services/UrlValidator.kt | 132 ++++++++++++++---- 7 files changed, 215 insertions(+), 46 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt diff --git a/openapi/paths/proxy.yaml b/openapi/paths/proxy.yaml index b1162a6a..2c7685cd 100644 --- a/openapi/paths/proxy.yaml +++ b/openapi/paths/proxy.yaml @@ -3,8 +3,10 @@ Proxy: tags: [extraction] summary: Retrieve proxied media description: >- - Streams supported remote content. Existing clients that submit a YouTube timed-text URL are - routed through the dedicated subtitle resolver for compatibility; new clients should use + Streams media from the supported YouTube, NicoNico, and BiliBili delivery hosts. Other + destinations, non-public addresses, cross-provider redirects, and non-HTTPS URLs are + rejected. Existing clients that submit a YouTube timed-text URL are routed through the + dedicated subtitle resolver for compatibility; new clients should use /subtitles/youtube/{videoId}. parameters: - name: url diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 35736ea9..3268ea8e 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -2,7 +2,6 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse -import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import java.io.ByteArrayInputStream @@ -10,10 +9,13 @@ import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 -internal fun readBilibiliRangeWithRetry(client: OkHttpClient, request: Request): ExtractionResult { +internal fun readBilibiliRangeWithRetry( + execute: (Request) -> Response, + request: Request, +): ExtractionResult { var lastMessage = "Proxy fetch failed" for (attempt in 1..BILIBILI_RANGE_ATTEMPTS) { - runCatching { client.newCall(request).execute() } + runCatching { execute(request) } .onSuccess { response -> response.use { val result = it.readBilibiliRangeBytes() diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index 15839531..cf12d6a9 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -13,11 +13,12 @@ import java.util.concurrent.ConcurrentHashMap class HlsManifestService( private val streamService: StreamService, - private val httpClient: OkHttpClient, + httpClient: OkHttpClient, cache: CacheService? = null, private val signManifestUrl: ((String) -> String)? = null, private val attestedYoutubeHls: suspend (String) -> String? = { null }, ) { + private val proxyHttp = ProxyHttpExecutor(httpClient) private val manifestCache = cache?.let(::HlsManifestCache) private val inFlight = ConcurrentHashMap>>() @@ -101,7 +102,7 @@ class HlsManifestService( .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) .apply { if (domandBid != null) header("Cookie", "domand_bid=$domandBid") } .build() - httpClient.newCall(request).execute() + proxyHttp.execute(request) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 6131c3ac..9c475dd1 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -40,13 +40,15 @@ internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: S } } -class NicoVideoProxyService { +class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { + private val executor = ProxyHttpExecutor(client) - private val client = OkHttpClient.Builder() + companion object { + private fun defaultNicoProxyClient() = OkHttpClient.Builder() .connectTimeout(10, TimeUnit.SECONDS) .readTimeout(30, TimeUnit.SECONDS) - .followRedirects(true) .build() + } suspend fun fetchManifest(rawUrl: String, domandBid: String? = null): ExtractionResult = withContext(Dispatchers.IO) { @@ -60,7 +62,7 @@ class NicoVideoProxyService { .url(manifestUrl) .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) if (resolvedBid != null) builder.header("Cookie", "domand_bid=$resolvedBid") - client.newCall(builder.build()).execute() + executor.execute(builder.build()) }.fold( onSuccess = { response -> val body = response.body @@ -95,7 +97,7 @@ class NicoVideoProxyService { .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) if (rangeHeader != null) builder.header("Range", rangeHeader) if (domandBid != null) builder.header("Cookie", "domand_bid=$domandBid") - client.newCall(builder.build()).execute() + executor.execute(builder.build()) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 27b4f1eb..ebde6bc6 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -21,7 +21,8 @@ internal fun rewriteHlsManifest(manifest: String): String = "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } -class OkHttpProxyService(private val client: OkHttpClient) : ProxyService { +class OkHttpProxyService(client: OkHttpClient) : ProxyService { + private val executor = ProxyHttpExecutor(client) override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = withContext(Dispatchers.IO) { @@ -44,8 +45,10 @@ class OkHttpProxyService(private val client: OkHttpClient) : ProxyService { if (resolvedDomandBid != null && isNicoNico(cleanUrl)) builder.header("Cookie", "domand_bid=$resolvedDomandBid") if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() - if (bilibili && rangeHeader != null) return@withContext readBilibiliRangeWithRetry(client, request) - client.newCall(request).execute() + if (bilibili && rangeHeader != null) { + return@withContext readBilibiliRangeWithRetry(executor::execute, request) + } + executor.execute(request) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt b/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt new file mode 100644 index 00000000..4f27c759 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt @@ -0,0 +1,89 @@ +package dev.typetype.server.services + +import okhttp3.Dns +import okhttp3.HttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.Proxy +import java.net.UnknownHostException +import java.util.concurrent.ConcurrentHashMap + +internal class ProxyHttpExecutor( + client: OkHttpClient, + private val maxRedirects: Int = 5, +) { + private val dns = ValidatingProxyDns(client.dns) + private val client = client.newBuilder() + .dns(dns) + .followRedirects(false) + .followSslRedirects(false) + .build() + + fun execute(initialRequest: Request): Response { + var request = initialRequest + val initialTarget = requireProxyTarget(request.url.toString()) + repeat(maxRedirects + 1) { redirectCount -> + val target = requireProxyTarget(request.url.toString()) + if (target.provider != initialTarget.provider) { + throw ProxyTargetRejectedException("Cross-provider redirect is not allowed") + } + trustConfiguredProxy(target.url) + dns.lookup(target.url.host) + val response = client.newCall(request).execute() + val location = response.header("Location") + if (!response.isRedirect || location == null) return response + if (redirectCount == maxRedirects) { + response.close() + throw IOException("Too many proxy redirects") + } + val nextUrl = response.request.url.resolve(location) + response.close() + if (nextUrl == null) throw ProxyTargetRejectedException("Invalid proxy redirect") + request = request.newBuilder().url(nextUrl).build() + } + throw IOException("Too many proxy redirects") + } + + private fun trustConfiguredProxy(target: HttpUrl) { + val configured = client.proxy?.let(::listOf) + ?: client.proxySelector.select(target.toUri()) + configured.forEach { proxy -> + if (proxy.type() == Proxy.Type.DIRECT) return@forEach + val address = proxy.address() as? InetSocketAddress ?: return@forEach + dns.trustTransportHost(address.hostString) + } + } +} + +internal class ValidatingProxyDns(private val delegate: Dns) : Dns { + private val trustedTransportHosts = ConcurrentHashMap.newKeySet() + + override fun lookup(hostname: String): List { + val normalized = hostname.lowercase().trimEnd('.') + val trustedTransport = normalized in trustedTransportHosts + if (!trustedTransport && providerForProxyHost(normalized) == null) { + throw UnknownHostException("Unsupported proxy host") + } + val addresses = try { + delegate.lookup(hostname) + } catch (error: UnknownHostException) { + throw error + } catch (error: Exception) { + throw UnknownHostException(error.message ?: "Unable to resolve proxy host") + } + if (addresses.isEmpty()) throw UnknownHostException("Unable to resolve proxy host") + if (trustedTransport) return addresses + if (addresses.any { !isPublicProxyAddress(it) }) { + throw UnknownHostException("Blocked non-public proxy address") + } + return addresses + } + + fun trustTransportHost(hostname: String) { + trustedTransportHosts += hostname.lowercase().trimEnd('.') + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt b/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt index 8785e81a..d7b488cc 100644 --- a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt +++ b/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt @@ -1,43 +1,113 @@ package dev.typetype.server.services +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.net.InetAddress import java.net.URI -private val BLOCKED_HOST_SUFFIXES = listOf(".local", ".internal", ".localhost") -private val PRIVATE_RANGES = listOf( - intArrayOf(10, 0, 0, 0) to 8, - intArrayOf(172, 16, 0, 0) to 12, - intArrayOf(192, 168, 0, 0) to 16, - intArrayOf(127, 0, 0, 0) to 8, - intArrayOf(169, 254, 0, 0) to 16, - intArrayOf(0, 0, 0, 0) to 8, +internal enum class ProxyProvider { + YOUTUBE, + BILIBILI, + NICONICO, +} + +internal data class ProxyTarget( + val url: HttpUrl, + val provider: ProxyProvider, +) + +internal class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) + +private val BLOCKED_IPV4_RANGES = listOf( + ipv4(0, 0, 0, 0) to 8, + ipv4(10, 0, 0, 0) to 8, + ipv4(100, 64, 0, 0) to 10, + ipv4(127, 0, 0, 0) to 8, + ipv4(169, 254, 0, 0) to 16, + ipv4(172, 16, 0, 0) to 12, + ipv4(192, 0, 0, 0) to 24, + ipv4(192, 0, 2, 0) to 24, + ipv4(192, 168, 0, 0) to 16, + ipv4(198, 18, 0, 0) to 15, + ipv4(198, 51, 100, 0) to 24, + ipv4(203, 0, 113, 0) to 24, + ipv4(224, 0, 0, 0) to 4, + ipv4(240, 0, 0, 0) to 4, ) -internal fun validateProxyUrl(raw: String): String? { - val uri = runCatching { URI(raw) }.getOrElse { return "Malformed URL" } - val scheme = uri.scheme?.lowercase() ?: return "Missing URL scheme" - if (scheme != "http" && scheme != "https") return "Unsupported URL scheme: $scheme" - val host = uri.host?.lowercase() ?: return "Missing URL host" - if (host == "localhost") return "Blocked host" - if (BLOCKED_HOST_SUFFIXES.any { host.endsWith(it) }) return "Blocked host" - val addr = runCatching { InetAddress.getByName(host) }.getOrElse { return null } - val bytes = addr.address - if (bytes.size != 4) return null - val octets = bytes.map { it.toInt() and 0xFF } - for ((prefix, bits) in PRIVATE_RANGES) { - if (isInRange(octets, prefix, bits)) return "Blocked private address" +private val BLOCKED_IPV6_RANGES = listOf( + byteArrayOf(0x20, 0x01, 0x00, 0x00) to 32, + byteArrayOf(0x20, 0x01, 0x00, 0x02, 0x00, 0x00) to 48, + byteArrayOf(0x20, 0x01, 0x00, 0x10) to 28, + byteArrayOf(0x20, 0x01, 0x00, 0x20) to 28, + byteArrayOf(0x20, 0x01, 0x0D, 0xB8.toByte()) to 32, + byteArrayOf(0x20, 0x02) to 16, +) + +internal fun validateProxyUrl(raw: String): String? = + runCatching { requireProxyTarget(raw) } + .exceptionOrNull() + ?.message + +internal fun requireProxyTarget(raw: String): ProxyTarget { + val uri = runCatching { URI(raw) }.getOrElse { throw ProxyTargetRejectedException("Malformed URL") } + val scheme = uri.scheme?.lowercase() ?: throw ProxyTargetRejectedException("Missing URL scheme") + if (scheme != "https") throw ProxyTargetRejectedException("Unsupported URL scheme: $scheme") + if (uri.rawUserInfo != null) throw ProxyTargetRejectedException("URL credentials are not allowed") + if (uri.host == null) throw ProxyTargetRejectedException("Missing URL host") + val url = raw.toHttpUrlOrNull() ?: throw ProxyTargetRejectedException("Malformed URL") + if (url.username.isNotEmpty() || url.password.isNotEmpty()) { + throw ProxyTargetRejectedException("URL credentials are not allowed") } - return null + if (url.port != 443) throw ProxyTargetRejectedException("Unsupported proxy port") + val provider = providerForProxyHost(url.host) + ?: throw ProxyTargetRejectedException("Unsupported proxy host") + return ProxyTarget(url, provider) } -private fun isInRange(octets: List, prefix: IntArray, bits: Int): Boolean { - var remaining = bits - for (i in prefix.indices) { - val maskBits = remaining.coerceIn(0, 8) - val mask = if (maskBits == 0) 0 else (0xFF shl (8 - maskBits)) and 0xFF - if ((octets[i] and mask) != (prefix[i] and mask)) return false - remaining -= maskBits - if (remaining <= 0) break +internal fun providerForProxyHost(rawHost: String): ProxyProvider? { + val host = rawHost.lowercase().trimEnd('.') + return when { + host.matchesHost("googlevideo.com") || + host.matchesHost("ytimg.com") || + host.matchesHost("ggpht.com") || + host == "yt3.googleusercontent.com" -> ProxyProvider.YOUTUBE + host.matchesHost("bilivideo.com") || + host.matchesHost("bilivideo.cn") || + host.matchesHost("hdslb.com") || + host == "upos-hz-mirrorakam.akamaized.net" -> ProxyProvider.BILIBILI + host.matchesHost("nicovideo.jp") || host.matchesHost("nimg.jp") -> ProxyProvider.NICONICO + else -> null } - return true +} + +internal fun isPublicProxyAddress(address: InetAddress): Boolean { + if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || + address.isSiteLocalAddress || address.isMulticastAddress + ) return false + val bytes = address.address + return when (bytes.size) { + 4 -> BLOCKED_IPV4_RANGES.none { (prefix, bits) -> hasPrefix(bytes, prefix, bits) } + 16 -> isPublicIpv6(bytes) + else -> false + } +} + +private fun isPublicIpv6(bytes: ByteArray): Boolean { + if ((bytes[0].toInt() and 0xE0) != 0x20) return false + return BLOCKED_IPV6_RANGES.none { (prefix, bits) -> hasPrefix(bytes, prefix, bits) } +} + +private fun String.matchesHost(suffix: String): Boolean = this == suffix || endsWith(".$suffix") + +private fun ipv4(a: Int, b: Int, c: Int, d: Int): ByteArray = + byteArrayOf(a.toByte(), b.toByte(), c.toByte(), d.toByte()) + +private fun hasPrefix(address: ByteArray, prefix: ByteArray, bits: Int): Boolean { + val fullBytes = bits / 8 + for (index in 0 until fullBytes) if (address[index] != prefix[index]) return false + val remaining = bits % 8 + if (remaining == 0) return true + val mask = 0xFF shl (8 - remaining) + return (address[fullBytes].toInt() and mask) == (prefix[fullBytes].toInt() and mask) } From ba098b8686e0bb8ca50dfebd4d98f9e9cf7e5d29 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:19 +0200 Subject: [PATCH 04/65] test: cover proxy destination validation --- .../typetype/server/BilibiliRangeProxyTest.kt | 8 +- .../server/HlsManifestServiceCacheTest.kt | 31 ++++--- .../dev/typetype/server/UrlValidatorTest.kt | 81 ++++++++++++++++--- 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt index 69b1a63f..796c59cc 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.OkHttpProxyService import kotlinx.coroutines.runBlocking import okhttp3.OkHttpClient +import okhttp3.Dns import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody @@ -11,6 +12,7 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import java.io.IOException +import java.net.InetAddress class BilibiliRangeProxyTest { @@ -18,7 +20,9 @@ class BilibiliRangeProxyTest { fun `BiliBili range proxy retries transport failures`() = runBlocking { var calls = 0 val bytes = byteArrayOf(1, 2, 3, 4) - val client = OkHttpClient.Builder().addInterceptor { chain -> + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor { chain -> calls += 1 val request = chain.request() assertEquals(OkHttpProxyService.BILIBILI_USER_AGENT, request.header("User-Agent")) @@ -36,7 +40,7 @@ class BilibiliRangeProxyTest { .header("Content-Range", "bytes 0-3/4") .body(bytes.toResponseBody()) .build() - }.build() + }.build() val service = OkHttpProxyService(client) val result = service.pipe( diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index 54b815c6..ada3a2a6 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.HlsManifestService import dev.typetype.server.services.StreamService import kotlinx.coroutines.test.runTest import okhttp3.Interceptor +import okhttp3.Dns import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Protocol @@ -15,12 +16,13 @@ import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.net.InetAddress class HlsManifestServiceCacheTest { @Test fun `hls manifests are cached briefly by manifest url`() = runTest { var calls = 0 - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> calls += 1 Response.Builder() .request(chain.request()) @@ -29,9 +31,9 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U\nsegment.ts".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) - val url = "https://example.com/master.m3u8" + val url = "https://manifest.googlevideo.com/master.m3u8" service.hlsManifest(url) service.hlsManifest(url) @@ -43,7 +45,7 @@ class HlsManifestServiceCacheTest { fun `attested manifest is scoped to youtube live`() = runTest { val requestedUrls = mutableListOf() val attestedVideoIds = mutableListOf() - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> requestedUrls += chain.request().url.toString() Response.Builder() .request(chain.request()) @@ -52,13 +54,13 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val streams = FixedStreamService( - testStreamResponse(hlsUrl = "https://example.com/legacy.m3u8").copy(isLive = true), + testStreamResponse(hlsUrl = "https://upos-hz-mirrorakam.akamaized.net/legacy.m3u8").copy(isLive = true), ) val service = HlsManifestService(streams, client, attestedYoutubeHls = { videoId -> attestedVideoIds += videoId - "https://example.com/attested.m3u8" + "https://manifest.googlevideo.com/attested.m3u8" }) val publicResult = service.hlsManifest("https://youtube.com/watch?v=test-id") @@ -73,9 +75,9 @@ class HlsManifestServiceCacheTest { assertEquals(listOf("test-id", "session-id"), attestedVideoIds) assertEquals( listOf( - "https://example.com/attested.m3u8", - "https://example.com/attested.m3u8", - "https://example.com/legacy.m3u8", + "https://manifest.googlevideo.com/attested.m3u8", + "https://manifest.googlevideo.com/attested.m3u8", + "https://upos-hz-mirrorakam.akamaized.net/legacy.m3u8", ), requestedUrls, ) @@ -84,7 +86,7 @@ class HlsManifestServiceCacheTest { @Test fun `NicoNico manifests use signed cookie and proxy segments`() = runTest { val requests = mutableListOf>() - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> requests += chain.request().url.toString() to chain.request().header("Cookie") Response.Builder() .request(chain.request()) @@ -93,7 +95,7 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U\nsegment.cmfa".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val service = HlsManifestService(NoopStreamService, client) val result = service.hlsManifest( @@ -110,6 +112,11 @@ class HlsManifestServiceCacheTest { } } +private fun proxyTestClient(interceptor: Interceptor): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(interceptor) + .build() + private object NoopStreamService : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult = ExtractionResult.Failure("unused") diff --git a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt b/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt index a0138de9..fc3d0252 100644 --- a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt @@ -1,32 +1,89 @@ package dev.typetype.server +import dev.typetype.server.services.ProxyProvider +import dev.typetype.server.services.isPublicProxyAddress +import dev.typetype.server.services.requireProxyTarget import dev.typetype.server.services.validateProxyUrl import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.net.InetAddress class UrlValidatorTest { @Test fun `rejects malformed and unsupported urls`() { assertEquals("Malformed URL", validateProxyUrl("not a url")) - assertEquals("Missing URL scheme", validateProxyUrl("example.com/video")) - assertEquals("Unsupported URL scheme: ftp", validateProxyUrl("ftp://example.com/video")) + assertEquals("Missing URL scheme", validateProxyUrl("i.ytimg.com/video")) + assertEquals("Unsupported URL scheme: http", validateProxyUrl("http://i.ytimg.com/video")) + assertEquals("Unsupported URL scheme: ftp", validateProxyUrl("ftp://i.ytimg.com/video")) assertEquals("Missing URL host", validateProxyUrl("https:///video")) + assertEquals("URL credentials are not allowed", validateProxyUrl("https://user@i.ytimg.com/video")) + assertEquals("Unsupported proxy port", validateProxyUrl("https://i.ytimg.com:8443/video")) } @Test - fun `blocks localhost and private ipv4 ranges`() { - assertEquals("Blocked host", validateProxyUrl("http://localhost/video")) - assertEquals("Blocked host", validateProxyUrl("https://demo.localhost/video")) - assertEquals("Blocked private address", validateProxyUrl("http://127.0.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://10.0.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://172.16.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://192.168.1.1/video")) + fun `allows only supported provider hosts`() { + assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://i.ytimg.com/image.jpg").provider) + assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://yt3.googleusercontent.com/avatar").provider) + assertEquals(ProxyProvider.BILIBILI, requireProxyTarget("https://i2.hdslb.com/image.jpg").provider) + assertEquals( + ProxyProvider.BILIBILI, + requireProxyTarget("https://upos-hz-mirrorakam.akamaized.net/video.m4s").provider, + ) + assertEquals( + ProxyProvider.NICONICO, + requireProxyTarget("https://delivery.domand.nicovideo.jp/video.m3u8").provider, + ) + assertNull(validateProxyUrl("https://r1---sn-a5mekn6z.googlevideo.com/videoplayback")) } @Test - fun `allows public and ipv6 addresses`() { - assertNull(validateProxyUrl("https://1.1.1.1/videoplayback?id=1")) - assertNull(validateProxyUrl("https://[2606:4700:4700::1111]/videoplayback?id=1")) + fun `rejects arbitrary and lookalike hosts`() { + assertEquals("Unsupported proxy host", validateProxyUrl("https://example.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://evilgooglevideo.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://googlevideo.com.example.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://example.googleusercontent.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://other.akamaized.net/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://127.0.0.1/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://[::1]/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://i.ytimg.com.evil.example/content")) + assertEquals("URL credentials are not allowed", validateProxyUrl("https://evil.example@i.ytimg.com/content")) + } + + @Test + fun `rejects non-public ipv4 addresses`() { + val blocked = listOf( + "0.0.0.1", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.168.1.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + ) + blocked.forEach { assertFalse(isPublicProxyAddress(InetAddress.getByName(it)), it) } + assertTrue(isPublicProxyAddress(InetAddress.getByName("1.1.1.1"))) + } + + @Test + fun `rejects non-public ipv6 addresses`() { + val blocked = listOf( + "::", + "::1", + "fc00::1", + "fe80::1", + "2001:10::1", + "2001:20::1", + "2001:db8::1", + "2002:7f00:1::", + "ff02::1", + ) + blocked.forEach { assertFalse(isPublicProxyAddress(InetAddress.getByName(it)), it) } + assertTrue(isPublicProxyAddress(InetAddress.getByName("2606:4700:4700::1111"))) } } From e75f1819559530b6025e685f49f22ef5c4855088 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:29 +0200 Subject: [PATCH 05/65] test: cover proxy redirect and dns attacks --- .../server/OkHttpProxyServiceSecurityTest.kt | 66 +++++++ .../typetype/server/ProxyHttpExecutorTest.kt | 167 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt b/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt new file mode 100644 index 00000000..3c1a162e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.OkHttpProxyService +import kotlinx.coroutines.test.runTest +import okhttp3.Dns +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class OkHttpProxyServiceSecurityTest { + @Test + fun `arbitrary destinations are rejected without a network call`() = runTest { + var calls = 0 + val client = OkHttpClient.Builder() + .addInterceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("private".toResponseBody()) + .build() + } + .build() + + val result = OkHttpProxyService(client).pipe( + url = "https://example.com/collect", + rangeHeader = null, + domandBid = null, + ) + + assertEquals(ExtractionResult.BadRequest("Unsupported proxy host"), result) + assertEquals(0, calls) + } + + @Test + fun `supported media hosts remain available`() = runTest { + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("image".toResponseBody()) + .build() + } + .build() + + val result = OkHttpProxyService(client).pipe( + url = "https://i.ytimg.com/vi/id/hqdefault.jpg", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt b/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt new file mode 100644 index 00000000..9a46d449 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt @@ -0,0 +1,167 @@ +package dev.typetype.server + +import dev.typetype.server.services.ProxyHttpExecutor +import dev.typetype.server.services.ValidatingProxyDns +import okhttp3.Dns +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress +import java.net.UnknownHostException + +class ProxyHttpExecutorTest { + @Test + fun `blocks private resolutions before sending a request`() { + var calls = 0 + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("127.0.0.1")) }) + .addInterceptor { chain -> + calls += 1 + ok(chain) + } + .build() + + assertThrows(UnknownHostException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals(0, calls) + } + + @Test + fun `blocks a redirect outside the original provider`() { + var calls = 0 + val client = testClient { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "https://example.com/collect") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(IllegalArgumentException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Unsupported proxy host", error.message) + assertEquals(1, calls) + } + + @Test + fun `follows bounded redirects inside one provider`() { + val requestedHosts = mutableListOf() + val client = testClient { chain -> + requestedHosts += chain.request().url.host + if (requestedHosts.size == 1) { + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "https://yt3.ggpht.com/avatar") + .body("".toResponseBody()) + .build() + } else { + ok(chain) + } + } + + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")).use { response -> + assertTrue(response.isSuccessful) + } + assertEquals(listOf("i.ytimg.com", "yt3.ggpht.com"), requestedHosts) + } + + @Test + fun `rejects mixed public and private dns answers`() { + val dns = ValidatingProxyDns( + Dns { + listOf( + InetAddress.getByName("1.1.1.1"), + InetAddress.getByName("10.0.0.1"), + ) + }, + ) + + assertThrows(UnknownHostException::class.java) { dns.lookup("i.ytimg.com") } + } + + @Test + fun `rejects a later private dns rebind answer`() { + var lookups = 0 + val dns = ValidatingProxyDns( + Dns { + lookups += 1 + listOf(InetAddress.getByName(if (lookups == 1) "1.1.1.1" else "10.0.0.1")) + }, + ) + + assertEquals(listOf(InetAddress.getByName("1.1.1.1")), dns.lookup("i.ytimg.com")) + assertThrows(UnknownHostException::class.java) { dns.lookup("i.ytimg.com") } + } + + @Test + fun `rejects an https downgrade redirect`() { + val client = testClient { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "http://i.ytimg.com/image.jpg") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(IllegalArgumentException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Unsupported URL scheme: http", error.message) + } + + @Test + fun `stops a redirect loop at the configured bound`() { + var calls = 0 + val client = testClient { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "/next") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(java.io.IOException::class.java) { + ProxyHttpExecutor(client, maxRedirects = 2).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Too many proxy redirects", error.message) + assertEquals(3, calls) + } + + private fun testClient(interceptor: Interceptor): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(interceptor) + .build() + + private fun request(url: String): Request = Request.Builder().url(url).build() + + private fun ok(chain: Interceptor.Chain): Response = Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("ok".toResponseBody()) + .build() +} From c646e1f5d2766be9a3862c918901bb1297311a08 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 12:14:13 +0200 Subject: [PATCH 06/65] fix: update vulnerable jsoup dependency --- build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index d4655639..3d141621 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -27,6 +27,11 @@ repositories { dependencies { implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) implementation(platform("io.netty:netty-bom:4.2.16.Final")) + constraints { + implementation("org.jsoup:jsoup:1.23.1") { + because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") + } + } implementation("io.ktor:ktor-server-core-jvm") implementation("io.ktor:ktor-server-netty-jvm") implementation("io.ktor:ktor-server-content-negotiation-jvm") From 25eb032f2e3893b7527ac21a09de9dbbfc28d49f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 12:50:04 +0200 Subject: [PATCH 07/65] chore: update server dependencies --- build.gradle.kts | 8 ++++---- gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3d141621..406be47e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ import java.time.Instant plugins { kotlin("jvm") version "2.4.10" kotlin("plugin.serialization") version "2.4.10" - id("io.ktor.plugin") version "3.5.1" + id("io.ktor.plugin") version "3.5.2" id("jacoco") } @@ -48,14 +48,14 @@ dependencies { implementation("org.json:json:20260719") implementation("com.squareup.okhttp3:okhttp:5.4.0") implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") - implementation("org.jetbrains.exposed:exposed-core:1.3.1") - implementation("org.jetbrains.exposed:exposed-jdbc:1.3.1") + implementation("org.jetbrains.exposed:exposed-core:1.4.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.2.1") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.6.0") - testImplementation("org.junit.jupiter:junit-jupiter:6.1.2") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a9db1155..69dd0d04 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.0-bin.zip networkTimeout=10000 retries=0 retryBackOffMs=500 From c32ad4162b4bab1a872f0b78edc1df53e08eef9e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:20:24 +0200 Subject: [PATCH 08/65] fix: require explicit local extractor path --- CONTRIBUTING.md | 6 ++++++ settings.gradle.kts | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96bcc2bb..b5417bc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,12 @@ PipePipe Client and PipePipeExtractor are the behavioral references for extracti When a defect is general to PipePipeExtractor, prefer contributing the correction upstream. Keep TypeType-specific behavior in this repository only when it belongs to the TypeType API or when the upstream API cannot express the required backend behavior cleanly. +Builds use the PipePipeExtractor revision pinned in `build.gradle.kts`. To deliberately test a local checkout instead, pass its path explicitly: + +```sh +./gradlew -PpipePipeExtractorPath=../PipePipeExtractor test +``` + ## Programming preferences - Prefer clear names and structure over explanatory comments, but comments are welcome whenever a contributor finds them useful. diff --git a/settings.gradle.kts b/settings.gradle.kts index 459e1da9..0a997ae6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,7 +1,13 @@ rootProject.name = "typetype-server" -val localPipePipeExtractor = file("../PipePipeExtractor") -if (localPipePipeExtractor.isDirectory) { +val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") + .orNull + ?.let { file(it) } + +if (localPipePipeExtractor != null) { + require(localPipePipeExtractor.isDirectory) { + "pipePipeExtractorPath must point to a PipePipeExtractor checkout" + } includeBuild(localPipePipeExtractor) { dependencySubstitution { substitute(module("com.github.InfinityLoop1308.PipePipeExtractor:extractor")) From 951bca9880e49c24bcd03f7e54f7152cec8e0e5d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:26 +0200 Subject: [PATCH 09/65] fix: normalize YouTube channel tab URLs --- .../server/services/AllowedChannelsService.kt | 21 +++++++++++++++++++ .../services/BlockedContentProfileTest.kt | 1 + 2 files changed, 22 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt index fba4aea1..222bb64a 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt @@ -11,6 +11,7 @@ import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll +import java.net.URI class AllowedChannelsService { suspend fun getChannels(userId: String): List = DatabaseFactory.query { @@ -90,3 +91,23 @@ internal fun normalizeChannelKey(value: String): String = value.trim() Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), "https://youtube.com", ) + .withoutYoutubeTab() + +private fun String.withoutYoutubeTab(): String { + val uri = runCatching { URI(this) }.getOrNull() ?: return this + if (!uri.host.equals("youtube.com", ignoreCase = true)) return this + val segments = uri.path.split('/').filter(String::isNotBlank) + if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this + val path = "/${segments.dropLast(1).joinToString("/")}" + return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() +} + +private val YOUTUBE_CHANNEL_TABS = setOf( + "featured", + "videos", + "shorts", + "streams", + "playlists", + "community", + "about", +) diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt index 198aa1d3..29869968 100644 --- a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt @@ -32,6 +32,7 @@ class BlockedContentProfileTest { ) assertTrue(profile.blocksChannel("https://m.youtube.com/@Example/", "Other")) + assertTrue(profile.blocksChannel("https://youtube.com/@Example/streams", "Other")) assertTrue(profile.blocksChannel("", "test channel")) } From 83f53a16198131568e152969db408be3e9d315f1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:36 +0200 Subject: [PATCH 10/65] feat: add RSS feed data model --- .../dev/typetype/server/db/DatabaseFactory.kt | 14 +++++ .../server/db/tables/RssFeedChannelsTable.kt | 9 +++ .../server/db/tables/RssFeedServicesTable.kt | 9 +++ .../server/db/tables/RssFeedsTable.kt | 26 ++++++++ .../server/db/tables/RssUserPoliciesTable.kt | 10 ++++ .../typetype/server/models/RssFeedModels.kt | 59 +++++++++++++++++++ .../dev/typetype/server/TestDatabase.kt | 8 +++ 7 files changed, 135 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index e99acb31..f964f6db 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -23,6 +23,10 @@ import dev.typetype.server.db.tables.AdminSettingsTable import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -73,6 +77,10 @@ object DatabaseFactory { YoutubeSessionPairingsTable, BugReportsTable, NotificationStatesTable, + RssFeedsTable, + RssFeedChannelsTable, + RssFeedServicesTable, + RssUserPoliciesTable, ) exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS name TEXT") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS thumbnail_url TEXT") @@ -95,6 +103,12 @@ object DatabaseFactory { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS caption_styles TEXT NOT NULL DEFAULT '{}'") exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_enabled BOOLEAN NOT NULL DEFAULT FALSE") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_public_base_url TEXT") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_max_feeds_per_user INTEGER NOT NULL DEFAULT 10") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_max_items INTEGER NOT NULL DEFAULT 50") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_minimum_poll_minutes INTEGER NOT NULL DEFAULT 5") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_rate_limit_per_minute INTEGER NOT NULL DEFAULT 30") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'user'") exec("ALTER TABLE blocked_videos ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt new file mode 100644 index 00000000..e7cb052c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedChannelsTable : Table("rss_feed_channels") { + val feedId = text("feed_id") + val channelUrl = text("channel_url") + override val primaryKey = PrimaryKey(feedId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt new file mode 100644 index 00000000..b02f88b2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedServicesTable : Table("rss_feed_services") { + val feedId = text("feed_id") + val serviceId = integer("service_id") + override val primaryKey = PrimaryKey(feedId, serviceId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt new file mode 100644 index 00000000..37c69f5c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt @@ -0,0 +1,26 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedsTable : Table("rss_feeds") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val tokenHash = text("token_hash") + val scope = text("scope") + val includeVideos = bool("include_videos") + val includeShorts = bool("include_shorts") + val includeLive = bool("include_live") + val includeUpcoming = bool("include_upcoming") + val enabled = bool("enabled").default(true) + val createdAt = long("created_at") + val updatedAt = long("updated_at") + val lastUsedAt = long("last_used_at").nullable() + + init { + index(false, userId) + index(false, createdAt) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt new file mode 100644 index 00000000..26994cca --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssUserPoliciesTable : Table("rss_user_policies") { + val userId = text("user_id") + val enabled = bool("enabled").default(true) + val updatedAt = long("updated_at") + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt b/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt new file mode 100644 index 00000000..e3ebbb5f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt @@ -0,0 +1,59 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class RssFeedRequest( + val name: String, + val scope: String = "all", + val channelUrls: List = emptyList(), + val serviceIds: List = listOf(0, 5, 6), + val includeVideos: Boolean = true, + val includeShorts: Boolean = true, + val includeLive: Boolean = true, + val includeUpcoming: Boolean = true, +) + +@Serializable +data class RssFeedItem( + val id: String, + val name: String, + val scope: String, + val channelUrls: List, + val serviceIds: List, + val includeVideos: Boolean, + val includeShorts: Boolean, + val includeLive: Boolean, + val includeUpcoming: Boolean, + val enabled: Boolean, + val createdAt: Long, + val updatedAt: Long, + val lastUsedAt: Long? = null, +) + +@Serializable +data class RssFeedSecretItem(val feed: RssFeedItem, val feedUrl: String) + +@Serializable +data class RssFeedEnabledRequest(val enabled: Boolean) + +@Serializable +data class RssUserPolicyRequest(val enabled: Boolean) + +@Serializable +data class AdminRssFeedItem( + val feed: RssFeedItem, + val userId: String, + val userName: String, + val userEmail: String, + val userRssEnabled: Boolean, + val userSuspended: Boolean, +) + +@Serializable +data class AdminRssFeedsPage( + val items: List, + val page: Int, + val limit: Int, + val total: Long, +) diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index f55367ae..1a22cc72 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -14,6 +14,10 @@ import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.db.tables.NotificationStatesTable import dev.typetype.server.db.tables.SearchHistoryTable @@ -88,6 +92,10 @@ object TestDatabase { value?.takeIf { it.isNotBlank() } ?: fallback fun truncateAll() = transaction { + RssFeedChannelsTable.deleteAll() + RssFeedServicesTable.deleteAll() + RssFeedsTable.deleteAll() + RssUserPoliciesTable.deleteAll() PlaylistVideosTable.deleteAll() PlaylistsTable.deleteAll() SavedPlaylistsTable.deleteAll() From a4fc41c76adcd1775319333004faf58ea5471cd8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:40 +0200 Subject: [PATCH 11/65] feat: add RSS feed persistence --- .../server/services/RssFeedRepository.kt | 178 ++++++++++++++++++ .../server/services/RssFeedRowMapper.kt | 31 +++ .../typetype/server/services/RssFeedSecret.kt | 23 +++ .../server/services/RssFeedSelections.kt | 23 +++ 4 files changed, 255 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt new file mode 100644 index 00000000..d8fab751 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt @@ -0,0 +1,178 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.RssFeedRequest +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.jetbrains.exposed.v1.jdbc.update + +internal data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) + +internal class RssFeedRepository { + suspend fun list(userId: String): List = DatabaseFactory.query { + val rows = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } + .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) + .toList() + val selections = loadRssFeedSelections(rows.map { it[RssFeedsTable.id] }) + rows.map { it.toStoredFeed(selections).item } + } + + suspend fun find(feedId: String): StoredRssFeed? = DatabaseFactory.query { + RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.singleOrNull()?.toStoredFeed() + } + + suspend fun createWithinLimit( + userId: String, + id: String, + tokenHash: String, + request: RssFeedRequest, + limit: Int, + ): RssFeedItem? = + DatabaseFactory.query { + TransactionManager.current().exec("SELECT pg_advisory_xact_lock(${userId.hashCode().toLong()})") + val count = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId }.count() + if (count >= limit) return@query null + val now = System.currentTimeMillis() + RssFeedsTable.insert { + it[RssFeedsTable.id] = id + it[RssFeedsTable.userId] = userId + it[name] = request.name + it[RssFeedsTable.tokenHash] = tokenHash + it[scope] = request.scope + it[includeVideos] = request.includeVideos + it[includeShorts] = request.includeShorts + it[includeLive] = request.includeLive + it[includeUpcoming] = request.includeUpcoming + it[enabled] = true + it[createdAt] = now + it[updatedAt] = now + } + replaceSelections(id, request) + RssFeedsTable.selectAll().where { RssFeedsTable.id eq id }.single().toStoredFeed().item + } + + suspend fun update(userId: String, feedId: String, request: RssFeedRequest): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[name] = request.name + it[scope] = request.scope + it[includeVideos] = request.includeVideos + it[includeShorts] = request.includeShorts + it[includeLive] = request.includeLive + it[includeUpcoming] = request.includeUpcoming + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) return@query null + replaceSelections(feedId, request) + RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun replaceToken(userId: String, feedId: String, tokenHash: String): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[RssFeedsTable.tokenHash] = tokenHash + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun setEnabled(userId: String, feedId: String, enabled: Boolean): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[RssFeedsTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun setEnabledByAdmin(feedId: String, enabled: Boolean): RssFeedItem? = DatabaseFactory.query { + val changed = RssFeedsTable.update({ RssFeedsTable.id eq feedId }) { + it[RssFeedsTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun delete(userId: String, feedId: String): Boolean = DatabaseFactory.query { + val owned = RssFeedsTable.selectAll().where { + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }.count() > 0 + if (!owned) return@query false + deleteFeed(feedId) + true + } + + suspend fun deleteByAdmin(feedId: String): Boolean = DatabaseFactory.query { + val exists = RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.count() > 0 + if (!exists) return@query false + deleteFeed(feedId) + true + } + + suspend fun touch(feedId: String, timestamp: Long) = DatabaseFactory.query { + RssFeedsTable.update({ RssFeedsTable.id eq feedId }) { it[lastUsedAt] = timestamp } + } + + suspend fun userEnabled(userId: String): Boolean = DatabaseFactory.query { + val active = UsersTable.selectAll().where { UsersTable.id eq userId } + .singleOrNull()?.get(UsersTable.suspended) == false + if (!active) return@query false + RssUserPoliciesTable.selectAll().where { RssUserPoliciesTable.userId eq userId } + .singleOrNull()?.get(RssUserPoliciesTable.enabled) ?: true + } + + suspend fun setUserEnabled(userId: String, enabled: Boolean) = DatabaseFactory.query { + val changed = RssUserPoliciesTable.update({ RssUserPoliciesTable.userId eq userId }) { + it[RssUserPoliciesTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) RssUserPoliciesTable.insert { + it[RssUserPoliciesTable.userId] = userId + it[RssUserPoliciesTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + } + + private fun replaceSelections(feedId: String, request: RssFeedRequest) { + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + request.channelUrls.forEach { url -> + RssFeedChannelsTable.insert { + it[RssFeedChannelsTable.feedId] = feedId + it[channelUrl] = url + } + } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + request.serviceIds.forEach { service -> + RssFeedServicesTable.insert { + it[RssFeedServicesTable.feedId] = feedId + it[serviceId] = service + } + } + } + + private fun deleteFeed(feedId: String) { + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + RssFeedsTable.deleteWhere { RssFeedsTable.id eq feedId } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt new file mode 100644 index 00000000..55421556 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt @@ -0,0 +1,31 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.models.RssFeedItem +import org.jetbrains.exposed.v1.core.ResultRow + +internal fun ResultRow.toStoredFeed(): StoredRssFeed = + toStoredFeed(loadRssFeedSelections(listOf(this[RssFeedsTable.id]))) + +internal fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { + val id = this[RssFeedsTable.id] + return StoredRssFeed( + item = RssFeedItem( + id = id, + name = this[RssFeedsTable.name], + scope = this[RssFeedsTable.scope], + channelUrls = selections.channels[id].orEmpty(), + serviceIds = selections.services[id].orEmpty(), + includeVideos = this[RssFeedsTable.includeVideos], + includeShorts = this[RssFeedsTable.includeShorts], + includeLive = this[RssFeedsTable.includeLive], + includeUpcoming = this[RssFeedsTable.includeUpcoming], + enabled = this[RssFeedsTable.enabled], + createdAt = this[RssFeedsTable.createdAt], + updatedAt = this[RssFeedsTable.updatedAt], + lastUsedAt = this[RssFeedsTable.lastUsedAt], + ), + userId = this[RssFeedsTable.userId], + tokenHash = this[RssFeedsTable.tokenHash], + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt new file mode 100644 index 00000000..563dda78 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt @@ -0,0 +1,23 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 + +internal class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { + fun create(): String { + val bytes = ByteArray(32) + random.nextBytes(bytes) + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) + } + + fun hash(secret: String): String = Base64.getUrlEncoder().withoutPadding().encodeToString( + MessageDigest.getInstance("SHA-256").digest(secret.toByteArray(Charsets.UTF_8)), + ) + + fun matches(secret: String, expectedHash: String): Boolean { + val actual = runCatching { Base64.getUrlDecoder().decode(hash(secret)) }.getOrNull() ?: return false + val expected = runCatching { Base64.getUrlDecoder().decode(expectedHash) }.getOrNull() ?: return false + return MessageDigest.isEqual(actual, expected) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt new file mode 100644 index 00000000..17343952 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt @@ -0,0 +1,23 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal data class RssFeedSelections( + val channels: Map>, + val services: Map>, +) + +internal fun loadRssFeedSelections(feedIds: List): RssFeedSelections { + if (feedIds.isEmpty()) return RssFeedSelections(emptyMap(), emptyMap()) + val channels = RssFeedChannelsTable.selectAll() + .where { RssFeedChannelsTable.feedId inList feedIds } + .groupBy({ it[RssFeedChannelsTable.feedId] }, { it[RssFeedChannelsTable.channelUrl] }) + val services = RssFeedServicesTable.selectAll() + .where { RssFeedServicesTable.feedId inList feedIds } + .groupBy({ it[RssFeedServicesTable.feedId] }, { it[RssFeedServicesTable.serviceId] }) + .mapValues { (_, values) -> values.sorted() } + return RssFeedSelections(channels, services) +} From bae862e255b824f4aaf96d286d3d6ad16542f4a0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:46 +0200 Subject: [PATCH 12/65] feat: configure RSS instance policy --- .../server/db/tables/AdminSettingsTable.kt | 6 ++ .../server/models/AdminSettingsItem.kt | 6 ++ .../server/models/InstanceResponse.kt | 10 +++ .../server/services/AdminSettingsService.kt | 62 ++++++++++++++++--- .../server/services/InstanceService.kt | 8 +++ .../server/AdminSettingsDefaultsTest.kt | 23 +++++++ .../dev/typetype/server/InstanceRoutesTest.kt | 15 +++++ 7 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt index 6590b798..e0703618 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt @@ -18,5 +18,11 @@ object AdminSettingsTable : Table("admin_settings") { val oidcAutoRedirect = bool("oidc_auto_redirect").default(false) val youtubeRemoteLoginEnabled = bool("youtube_remote_login_enabled").default(false) val accessMode = text("access_mode").default("unrestricted") + val rssEnabled = bool("rss_enabled").default(false) + val rssPublicBaseUrl = text("rss_public_base_url").nullable() + val rssMaxFeedsPerUser = integer("rss_max_feeds_per_user").default(10) + val rssMaxItems = integer("rss_max_items").default(50) + val rssMinimumPollMinutes = integer("rss_minimum_poll_minutes").default(5) + val rssRateLimitPerMinute = integer("rss_rate_limit_per_minute").default(30) override val primaryKey = PrimaryKey(id) } diff --git a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt index b58d80c1..08b60e63 100644 --- a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt @@ -18,4 +18,10 @@ data class AdminSettingsItem( val oidcAutoRedirect: Boolean = false, val youtubeRemoteLoginEnabled: Boolean = false, val accessMode: String = "unrestricted", + val rssEnabled: Boolean = false, + val rssPublicBaseUrl: String? = null, + val rssMaxFeedsPerUser: Int = 10, + val rssMaxItems: Int = 50, + val rssMinimumPollMinutes: Int = 5, + val rssRateLimitPerMinute: Int = 30, ) diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 2ec0c25a..55d8916d 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,4 +24,14 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, + val rss: RssInstanceCapability = RssInstanceCapability(), +) + +@Serializable +data class RssInstanceCapability( + val enabled: Boolean = false, + val maxFeedsPerUser: Int = 0, + val maxItems: Int = 0, + val minimumPollMinutes: Int = 0, + val rateLimitPerMinute: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt b/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt index 16495092..7a761a05 100644 --- a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt @@ -8,6 +8,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update +import java.net.URI private const val SETTINGS_ROW_ID = 1 @@ -33,6 +34,12 @@ class AdminSettingsService( oidcAutoRedirect = it[AdminSettingsTable.oidcAutoRedirect], youtubeRemoteLoginEnabled = it[AdminSettingsTable.youtubeRemoteLoginEnabled], accessMode = it[AdminSettingsTable.accessMode].toAccessMode(), + rssEnabled = it[AdminSettingsTable.rssEnabled], + rssPublicBaseUrl = it[AdminSettingsTable.rssPublicBaseUrl], + rssMaxFeedsPerUser = it[AdminSettingsTable.rssMaxFeedsPerUser], + rssMaxItems = it[AdminSettingsTable.rssMaxItems], + rssMinimumPollMinutes = it[AdminSettingsTable.rssMinimumPollMinutes], + rssRateLimitPerMinute = it[AdminSettingsTable.rssRateLimitPerMinute], ).normalized() } ?: defaultSettings().normalized() } @@ -59,6 +66,12 @@ class AdminSettingsService( it[oidcAutoRedirect] = settings.oidcAutoRedirect it[youtubeRemoteLoginEnabled] = settings.youtubeRemoteLoginEnabled it[accessMode] = settings.accessMode.toAccessMode() + it[rssEnabled] = settings.rssEnabled + it[rssPublicBaseUrl] = settings.rssPublicBaseUrl + it[rssMaxFeedsPerUser] = settings.rssMaxFeedsPerUser + it[rssMaxItems] = settings.rssMaxItems + it[rssMinimumPollMinutes] = settings.rssMinimumPollMinutes + it[rssRateLimitPerMinute] = settings.rssRateLimitPerMinute } } else { AdminSettingsTable.insert { @@ -76,6 +89,12 @@ class AdminSettingsService( it[oidcAutoRedirect] = settings.oidcAutoRedirect it[youtubeRemoteLoginEnabled] = settings.youtubeRemoteLoginEnabled it[accessMode] = settings.accessMode.toAccessMode() + it[rssEnabled] = settings.rssEnabled + it[rssPublicBaseUrl] = settings.rssPublicBaseUrl + it[rssMaxFeedsPerUser] = settings.rssMaxFeedsPerUser + it[rssMaxItems] = settings.rssMaxItems + it[rssMinimumPollMinutes] = settings.rssMinimumPollMinutes + it[rssRateLimitPerMinute] = settings.rssRateLimitPerMinute } } } @@ -83,17 +102,44 @@ class AdminSettingsService( return settings } - private fun AdminSettingsItem.normalized(): AdminSettingsItem = copy( - name = name.trim().takeIf { it.isNotEmpty() } ?: DEFAULT_INSTANCE_NAME, - tagline = tagline.normalizeOptionalText(), - logoUrl = logoUrl.normalizeOptionalText(), - bannerUrl = bannerUrl.normalizeOptionalText(), - minAndroidClientVersion = minAndroidClientVersion.normalizeOptionalText(), - accessMode = accessMode.toAccessMode(), - ) + private fun AdminSettingsItem.normalized(): AdminSettingsItem { + val publicBaseUrl = rssPublicBaseUrl.normalizePublicBaseUrl() + require(!rssEnabled || publicBaseUrl != null) { + "RSS public base URL is required when RSS is enabled" + } + return copy( + name = name.trim().takeIf { it.isNotEmpty() } ?: DEFAULT_INSTANCE_NAME, + tagline = tagline.normalizeOptionalText(), + logoUrl = logoUrl.normalizeOptionalText(), + bannerUrl = bannerUrl.normalizeOptionalText(), + minAndroidClientVersion = minAndroidClientVersion.normalizeOptionalText(), + accessMode = accessMode.toAccessMode(), + rssPublicBaseUrl = publicBaseUrl, + rssMaxFeedsPerUser = rssMaxFeedsPerUser.coerceIn(1, 100), + rssMaxItems = rssMaxItems.coerceIn(1, 200), + rssMinimumPollMinutes = rssMinimumPollMinutes.coerceIn(1, 1_440), + rssRateLimitPerMinute = rssRateLimitPerMinute.coerceIn(1, 600), + ) + } private fun String?.normalizeOptionalText(): String? = this?.trim()?.takeIf { it.isNotEmpty() } + private fun String?.normalizePublicBaseUrl(): String? { + val value = normalizeOptionalText() ?: return null + val uri = runCatching { URI(value) }.getOrNull() + ?: throw IllegalArgumentException("RSS public base URL must be an absolute HTTP or HTTPS URL") + require(uri.scheme in setOf("http", "https") && !uri.host.isNullOrBlank()) { + "RSS public base URL must be an absolute HTTP or HTTPS URL" + } + require(uri.rawQuery == null && uri.rawFragment == null) { + "RSS public base URL cannot contain a query or fragment" + } + require(uri.rawUserInfo == null) { + "RSS public base URL cannot contain credentials" + } + return value.trimEnd('/') + } + companion object { @Volatile private var cachedSettings: AdminSettingsItem? = null diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index 07e6bfa7..3225e2d6 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -7,6 +7,7 @@ import dev.typetype.server.models.InstanceMinClientVersion import dev.typetype.server.models.InstanceResponse import dev.typetype.server.models.OidcPublicConfig import dev.typetype.server.models.YoutubeRemoteLoginStatus +import dev.typetype.server.models.RssInstanceCapability class InstanceService( private val authService: AuthService, @@ -47,6 +48,13 @@ class InstanceService( youtubeRemoteLoginEnabled = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginReady = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginUnavailableReason = youtubeRemoteLoginStatus.unavailableReason, + rss = RssInstanceCapability( + enabled = settings.rssEnabled && settings.rssPublicBaseUrl != null, + maxFeedsPerUser = settings.rssMaxFeedsPerUser, + maxItems = settings.rssMaxItems, + minimumPollMinutes = settings.rssMinimumPollMinutes, + rateLimitPerMinute = settings.rssRateLimitPerMinute, + ), ) } diff --git a/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt b/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt index bc0d8def..fc35b7fa 100644 --- a/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt +++ b/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.AdminSettingsItem import dev.typetype.server.services.AdminSettingsService import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -27,4 +28,26 @@ class AdminSettingsDefaultsTest { assertEquals(false, settings.get().youtubeRemoteLoginEnabled) } + + @Test + fun `RSS public URL rejects embedded credentials`() = runTest { + val error = runCatching { + AdminSettingsService().upsert( + AdminSettingsItem(rssPublicBaseUrl = "https://user:password@video.example"), + ) + }.exceptionOrNull() + + assertTrue(error is IllegalArgumentException) + assertEquals("RSS public base URL cannot contain credentials", error?.message) + } + + @Test + fun `RSS cannot be enabled without a public URL`() = runTest { + val error = runCatching { + AdminSettingsService().upsert(AdminSettingsItem(rssEnabled = true)) + }.exceptionOrNull() + + assertTrue(error is IllegalArgumentException) + assertEquals("RSS public base URL is required when RSS is enabled", error?.message) + } } diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index 3881e19a..7d2eeb22 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -73,6 +73,9 @@ class InstanceRoutesTest { assertEquals(false, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) + val rss = root["rss"]?.jsonObject + assertEquals(false, rss?.get("enabled")?.jsonPrimitive?.boolean) + assertEquals(10, rss?.get("maxFeedsPerUser")?.jsonPrimitive?.int) assertEquals(listOf(0, 3, 4, 5, 6), root["supportedServices"]?.jsonArray?.map { it.jsonPrimitive.int }) assertEquals(null, root["androidPlayback"]) } @@ -90,6 +93,12 @@ class InstanceRoutesTest { localLoginEnabled = false, oidcAutoRedirect = true, youtubeRemoteLoginEnabled = true, + rssEnabled = true, + rssPublicBaseUrl = "https://video.example/", + rssMaxFeedsPerUser = 4, + rssMaxItems = 80, + rssMinimumPollMinutes = 15, + rssRateLimitPerMinute = 12, ) ) val auth = AuthService.fixed(TEST_USER_ID, hasUsers = true) @@ -115,6 +124,12 @@ class InstanceRoutesTest { assertEquals(true, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals(null, root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) + val rss = root["rss"]!!.jsonObject + assertEquals(true, rss["enabled"]?.jsonPrimitive?.boolean) + assertEquals(4, rss["maxFeedsPerUser"]?.jsonPrimitive?.int) + assertEquals(80, rss["maxItems"]?.jsonPrimitive?.int) + assertEquals(15, rss["minimumPollMinutes"]?.jsonPrimitive?.int) + assertEquals(12, rss["rateLimitPerMinute"]?.jsonPrimitive?.int) val register = client.post("/auth/register") { contentType(ContentType.Application.Json) setBody("""{"email":"new@test.local","password":"secret","name":"New"}""") From a5d277cc2d8b1f239cab27a5a9c8632dd3faa5a8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:49 +0200 Subject: [PATCH 13/65] feat: manage private RSS feeds --- .../server/services/RssFeedAdminRepository.kt | 46 ++++++ .../services/RssFeedManagementService.kt | 135 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt new file mode 100644 index 00000000..76816a96 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminRssFeedItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal class RssFeedAdminRepository { + suspend fun list(page: Int, limit: Int): Pair, Long> = DatabaseFactory.query { + val total = RssFeedsTable.selectAll().count() + val rows = RssFeedsTable.selectAll() + .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) + .limit(limit) + .offset((page - 1L) * limit) + .toList() + val selections = loadRssFeedSelections(rows.map { it[RssFeedsTable.id] }) + val feeds = rows.map { it.toStoredFeed(selections) } + val userIds = feeds.map { it.userId }.distinct() + val users = if (userIds.isEmpty()) emptyMap() else UsersTable.selectAll() + .where { UsersTable.id inList userIds } + .associateBy { it[UsersTable.id] } + val policies = if (userIds.isEmpty()) emptyMap() else RssUserPoliciesTable.selectAll() + .where { RssUserPoliciesTable.userId inList userIds } + .associate { it[RssUserPoliciesTable.userId] to it[RssUserPoliciesTable.enabled] } + feeds.mapNotNull { stored -> + val user = users[stored.userId] ?: return@mapNotNull null + AdminRssFeedItem( + feed = stored.item, + userId = stored.userId, + userName = user[UsersTable.name], + userEmail = user[UsersTable.email], + userRssEnabled = policies[stored.userId] ?: true, + userSuspended = user[UsersTable.suspended], + ) + } to total + } + + suspend fun userExists(userId: String): Boolean = DatabaseFactory.query { + UsersTable.selectAll().where { UsersTable.id eq userId }.any() + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt new file mode 100644 index 00000000..868f6a30 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt @@ -0,0 +1,135 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AdminRssFeedsPage +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.models.RssFeedSecretItem +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.util.UUID + +class RssFeedManagementService internal constructor( + private val settings: AdminSettingsService, + private val subscriptions: SubscriptionsService, + private val repository: RssFeedRepository = RssFeedRepository(), + private val adminRepository: RssFeedAdminRepository = RssFeedAdminRepository(), + private val secrets: RssFeedSecret = RssFeedSecret(), +) { + suspend fun list(userId: String): List { + requireAvailable(userId) + return repository.list(userId) + } + + suspend fun create(userId: String, request: RssFeedRequest): RssFeedSecretItem { + val config = requireAvailable(userId) + val normalized = normalize(userId, request) + val secret = secrets.create() + val feed = repository.createWithinLimit( + userId, + UUID.randomUUID().toString(), + secrets.hash(secret), + normalized, + config.rssMaxFeedsPerUser, + ) ?: throw RssFeedException("RSS feed limit reached", "rss_feed_limit_reached") + return RssFeedSecretItem(feed, feedUrl(config.rssPublicBaseUrl!!, feed.id, secret)) + } + + suspend fun update(userId: String, feedId: String, request: RssFeedRequest): RssFeedItem { + requireAvailable(userId) + val normalized = normalize(userId, request) + return repository.update(userId, feedId, normalized) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun setEnabled(userId: String, feedId: String, enabled: Boolean): RssFeedItem { + requireAvailable(userId) + return repository.setEnabled(userId, feedId, enabled) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun regenerate(userId: String, feedId: String): RssFeedSecretItem { + val config = requireAvailable(userId) + val secret = secrets.create() + val feed = repository.replaceToken(userId, feedId, secrets.hash(secret)) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + return RssFeedSecretItem(feed, feedUrl(config.rssPublicBaseUrl!!, feed.id, secret)) + } + + suspend fun delete(userId: String, feedId: String) { + requireAvailable(userId) + if (!repository.delete(userId, feedId)) throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun adminList(page: Int, limit: Int): AdminRssFeedsPage { + val (items, total) = adminRepository.list(page, limit) + return AdminRssFeedsPage(items, page, limit, total) + } + + suspend fun adminSetEnabled(feedId: String, enabled: Boolean): RssFeedItem = + repository.setEnabledByAdmin(feedId, enabled) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + + suspend fun adminDelete(feedId: String) { + if (!repository.deleteByAdmin(feedId)) throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun adminSetUserEnabled(userId: String, enabled: Boolean) { + if (!adminRepository.userExists(userId)) { + throw RssFeedException("User not found", "rss_user_not_found") + } + repository.setUserEnabled(userId, enabled) + } + + private suspend fun requireAvailable(userId: String) = settings.get().also { config -> + if (!config.rssEnabled || config.rssPublicBaseUrl == null) { + throw RssFeedException("RSS feeds are disabled", "rss_disabled") + } + if (!repository.userEnabled(userId)) throw RssFeedException("RSS feeds are disabled for this account", "rss_user_disabled") + } + + private suspend fun normalize(userId: String, request: RssFeedRequest): RssFeedRequest { + val name = request.name.trim() + if (name.length !in 1..100) throw RssFeedException("Name must contain 1 to 100 characters", "rss_invalid_name") + if (request.scope !in SCOPES) throw RssFeedException("Invalid RSS scope", "rss_invalid_scope") + val services = request.serviceIds.distinct().sorted() + if (services.isEmpty() || services.any { it !in SERVICES }) { + throw RssFeedException("Select at least one supported service", "rss_invalid_services") + } + if (!request.hasSelectedType()) throw RssFeedException("Select at least one content type", "rss_invalid_types") + val channels = when (request.scope) { + "all" -> emptyList() + "channels" -> validateChannels(userId, request.channelUrls) + else -> throw RssFeedException("Invalid RSS scope", "rss_invalid_scope") + } + return request.copy( + name = name, + channelUrls = channels, + serviceIds = services, + ) + } + + private suspend fun validateChannels(userId: String, rawChannels: List): List { + val channels = rawChannels.map(ChannelUrlCanonicalizer::canonicalize).filter(String::isNotBlank).distinct() + if (channels.isEmpty() || channels.size > 100) { + throw RssFeedException("Select between 1 and 100 subscribed channels", "rss_invalid_channels") + } + val subscribed = subscriptions.getAll(userId).map { it.channelUrl }.toSet() + if (channels.any { it !in subscribed }) { + throw RssFeedException("RSS channels must belong to your subscriptions", "rss_channel_not_subscribed") + } + return channels.sorted() + } + + private fun feedUrl(baseUrl: String, feedId: String, secret: String): String = + "$baseUrl/api/rss/feeds/$feedId.xml?token=${URLEncoder.encode(secret, StandardCharsets.UTF_8)}" + + private fun RssFeedRequest.hasSelectedType(): Boolean = + includeVideos || includeShorts || includeLive || includeUpcoming + + companion object { + private val SCOPES = setOf("all", "channels") + private val SERVICES = setOf(0, 5, 6) + } +} + +class RssFeedException(message: String, val code: String) : IllegalArgumentException(message) From c2d06fc6b6f89c64ac5819c79bc45b422f41fdbe Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:53 +0200 Subject: [PATCH 14/65] feat: render private RSS feeds --- .../server/services/RssDocumentRenderer.kt | 67 +++++++++++++++++++ .../server/services/RssFeedReaderService.kt | 60 +++++++++++++++++ .../server/services/RssFeedThrottle.kt | 29 ++++++++ .../server/services/RssVideoMetadata.kt | 20 ++++++ .../server/services/RssVideoTypeFilter.kt | 16 +++++ .../services/SubscriptionFeedService.kt | 3 + 6 files changed, 195 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt new file mode 100644 index 00000000..c84e9665 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem +import java.io.ByteArrayOutputStream +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import javax.xml.stream.XMLOutputFactory +import javax.xml.stream.XMLStreamWriter + +internal object RssDocumentRenderer { + fun render( + feed: RssFeedItem, + videos: List, + publicBaseUrl: String, + lastModified: Long, + ): ByteArray { + val output = ByteArrayOutputStream() + val writer = XMLOutputFactory.newFactory().createXMLStreamWriter(output, StandardCharsets.UTF_8.name()) + writer.writeStartDocument(StandardCharsets.UTF_8.name(), "1.0") + writer.writeStartElement("rss") + writer.writeAttribute("version", "2.0") + writer.writeStartElement("channel") + writer.element("title", feed.name) + writer.element("link", publicBaseUrl) + writer.element("description", "TypeType subscription feed: ${feed.name}") + writer.element("generator", "TypeType") + writer.element("lastBuildDate", RFC_1123.format(Instant.ofEpochMilli(lastModified))) + videos.forEach { writer.item(it, publicBaseUrl) } + writer.writeEndElement() + writer.writeEndElement() + writer.writeEndDocument() + writer.close() + return output.toByteArray() + } + + fun lastModified(feed: RssFeedItem, videos: List, now: Long): Long = + maxOf(feed.updatedAt, videos.maxOfOrNull(RssVideoMetadata::publishedAtMillis) ?: feed.updatedAt) + .coerceAtMost(now) + + private fun XMLStreamWriter.item(video: VideoItem, publicBaseUrl: String) { + val watchUrl = "$publicBaseUrl/watch?v=${URLEncoder.encode(video.url, StandardCharsets.UTF_8)}" + writeStartElement("item") + element("title", video.title) + element("link", watchUrl) + writeStartElement("guid") + writeAttribute("isPermaLink", "false") + writeCharacters("${RssVideoMetadata.serviceId(video)}:${video.id}") + writeEndElement() + element("author", video.uploaderName) + video.shortDescription?.takeIf(String::isNotBlank)?.let { element("description", it) } + RssVideoMetadata.publishedAtMillis(video).takeIf { it > 0 } + ?.let { element("pubDate", RFC_1123.format(Instant.ofEpochMilli(it))) } + writeEndElement() + } + + private fun XMLStreamWriter.element(name: String, value: String) { + writeStartElement(name) + writeCharacters(value) + writeEndElement() + } + + private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt new file mode 100644 index 00000000..eae437df --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt @@ -0,0 +1,60 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem +import java.security.MessageDigest +import java.util.HexFormat + +class RssFeedReaderService internal constructor( + private val settings: AdminSettingsService, + private val subscriptionFeed: SubscriptionFeedService, + private val blocked: BlockedService, + private val repository: RssFeedRepository = RssFeedRepository(), + private val throttle: RssFeedThrottle = RssFeedThrottle(), + private val secrets: RssFeedSecret = RssFeedSecret(), + private val clock: () -> Long = System::currentTimeMillis, +) { + suspend fun read(feedId: String, secret: String): RssFeedReadResult { + val config = settings.get() + val baseUrl = config.rssPublicBaseUrl + if (!config.rssEnabled || baseUrl == null) return RssFeedReadResult.NotFound + val stored = repository.find(feedId) ?: return RssFeedReadResult.NotFound + if (!stored.item.enabled || !repository.userEnabled(stored.userId)) return RssFeedReadResult.NotFound + if (!secrets.matches(secret, stored.tokenHash)) return RssFeedReadResult.NotFound + throttle.acquire(feedId, config.rssRateLimitPerMinute)?.let { return RssFeedReadResult.Throttled(it) } + + val scopeChannels = stored.item.channelUrls.takeIf { stored.item.scope == "channels" }?.toSet() + val profile = blocked.profileFor(stored.userId) + val now = clock() + val videos = subscriptionFeed.getCachedAll(stored.userId).orEmpty() + .asSequence() + .filter { scopeChannels == null || ChannelUrlCanonicalizer.canonicalize(it.uploaderUrl) in scopeChannels } + .filter { RssVideoMetadata.serviceId(it) in stored.item.serviceIds } + .filter { RssVideoTypeFilter.includes(stored.item, it, now) } + .filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) } + .take(config.rssMaxItems) + .toList() + val lastModified = RssDocumentRenderer.lastModified(stored.item, videos, now) + val bytes = RssDocumentRenderer.render(stored.item, videos, baseUrl, lastModified) + val etag = "\"${HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes))}\"" + if (stored.item.lastUsedAt == null || now - stored.item.lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { + repository.touch(feedId, now) + } + return RssFeedReadResult.Ready(bytes, etag, lastModified, config.rssMinimumPollMinutes * 60) + } + + companion object { + private const val LAST_USED_WRITE_INTERVAL_MS = 60_000L + } +} + +sealed interface RssFeedReadResult { + data class Ready( + val bytes: ByteArray, + val etag: String, + val lastModified: Long, + val maxAgeSeconds: Int, + ) : RssFeedReadResult + + data class Throttled(val retryAfterSeconds: Int) : RssFeedReadResult + data object NotFound : RssFeedReadResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt new file mode 100644 index 00000000..87e81317 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.services + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +internal class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { + private val windows = ConcurrentHashMap() + private val acquisitions = AtomicInteger() + + fun acquire(feedId: String, limit: Int): Int? { + val now = clock() + if (acquisitions.incrementAndGet() % CLEANUP_INTERVAL == 0) { + windows.entries.removeIf { now - it.value.startedAt >= RETENTION_MS } + } + val window = windows.compute(feedId) { _, current -> + if (current == null || now - current.startedAt >= WINDOW_MS) Window(now, 1) else current.copy(count = current.count + 1) + }!! + if (window.count <= limit) return null + return ((WINDOW_MS - (now - window.startedAt) + 999L) / 1_000L).coerceAtLeast(1L).toInt() + } + + private data class Window(val startedAt: Long, val count: Int) + + companion object { + private const val WINDOW_MS = 60_000L + private const val RETENTION_MS = WINDOW_MS * 2 + private const val CLEANUP_INTERVAL = 256 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt new file mode 100644 index 00000000..047eaaab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem +import java.net.URI + +internal object RssVideoMetadata { + fun serviceId(video: VideoItem): Int { + val host = runCatching { URI(video.url).host.orEmpty().lowercase() }.getOrDefault("") + return when { + host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> 5 + host == "nico.ms" || host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") -> 6 + else -> 0 + } + } + + fun publishedAtMillis(video: VideoItem): Long { + val value = video.publishedAt?.takeIf { it > 0 } ?: video.uploaded.takeIf { it > 0 } ?: 0L + return if (value in 1..9_999_999_999L) value * 1_000L else value + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt new file mode 100644 index 00000000..1e4e3dc6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem + +internal object RssVideoTypeFilter { + fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { + video.isLive -> feed.includeLive + isUpcoming(video, now) -> feed.includeUpcoming + video.isShortFormContent -> feed.includeShorts + else -> feed.includeVideos + } + + private fun isUpcoming(video: VideoItem, now: Long): Boolean = + !video.isPostLive && video.duration < 0 && RssVideoMetadata.publishedAtMillis(video) > now +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 48a09fc2..602c634e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -80,6 +80,9 @@ class SubscriptionFeedService( return snapshot.page(page * limit, limit, isRefreshing(userId)) } + internal suspend fun getCachedAll(userId: String): List? = + store.current(userId)?.videos + suspend fun invalidate(userId: String) { runCatching { store.invalidate(userId, UUID.randomUUID().toString()) } .onFailure { logger.warn("subscription_feed event=invalidate_failed user={} error={}", userKey(userId), it.message) } From 8ec0fffd45a0b9ceac18780146cceeabe224ff8f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:57 +0200 Subject: [PATCH 15/65] feat: expose private RSS feed API --- .../dev/typetype/server/ApplicationRoutes.kt | 4 + .../dev/typetype/server/ServiceRegistry.kt | 11 ++ .../typetype/server/routes/AdminRssRoutes.kt | 76 ++++++++++ .../typetype/server/routes/RssFeedRoutes.kt | 141 ++++++++++++++++++ .../typetype/server/routes/UserDataRoutes.kt | 1 + 5 files changed, 233 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 44d80b01..41e04025 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.routes.adminBugReportRoutes import dev.typetype.server.routes.adminAllowListRoutes import dev.typetype.server.routes.adminRoutes +import dev.typetype.server.routes.adminRssRoutes import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes @@ -17,6 +18,7 @@ import dev.typetype.server.routes.oidcAuthRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes +import dev.typetype.server.routes.rssPublicRoutes import dev.typetype.server.routes.sabrRoutes import dev.typetype.server.routes.searchRoutes import dev.typetype.server.routes.sessionActivityRoutes @@ -66,6 +68,7 @@ internal fun Application.installApplicationRoutes( routing { internalObservabilityRoutes(internalHealthService::check) publicMetadataRoutes(instanceService::getInstance) + rssPublicRoutes(svc.rssFeedReaderService) installStreamRoutes(svc, authService, adminSettingsService) rateLimit(DEARROW_ZONE) { deArrowRoutes(svc.deArrowService) } rateLimit(EXTRACTION_ZONE) { @@ -102,6 +105,7 @@ internal fun Application.installApplicationRoutes( authSessionConfig, ) adminRoutes(authService, userAdminService, passwordResetService, adminSettingsService) + adminRssRoutes(svc.rssFeedManagementService, authService) adminIdentityRoutes(svc.accountIdentityService, authService) adminAllowListRoutes(authService, userAdminService, svc.adminManagedAccessService, svc.adminUserLookupService, svc.allowedChannelsService, svc.allowedPlaylistsService) adminSessionRoutes(authService, activeSessionService) diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index b0cff887..a6ee84ed 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -18,6 +18,8 @@ import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.NotificationsService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.ProgressService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReaderService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.SavedPlaylistService import dev.typetype.server.services.SearchHistoryService @@ -107,6 +109,15 @@ internal class ServiceRegistry( val adminUserLookupService = AdminUserLookupService() val accessControlService = AccessControlService(settingsService, allowedChannelsService, allowedPlaylistsService, adminSettingsService) val blockedService = BlockedService() + val rssFeedManagementService = RssFeedManagementService( + adminSettingsService, + subscriptionsService, + ) + val rssFeedReaderService = RssFeedReaderService( + adminSettingsService, + subscriptionFeedService, + blockedService, + ) val typeTypeBackupService = TypeTypeBackupService( subscriptionsService, historyService, diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt new file mode 100644 index 00000000..75f4fc2c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt @@ -0,0 +1,76 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.RssFeedEnabledRequest +import dev.typetype.server.models.RssUserPolicyRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.put + +fun Route.adminRssRoutes(service: RssFeedManagementService, authService: AuthService) { + get("/admin/rss/feeds") { + call.withRssAdmin(authService) { + val pageRaw = call.request.queryParameters["page"] + val limitRaw = call.request.queryParameters["limit"] + if ( + (pageRaw != null && pageRaw.toIntOrNull() == null) || + (limitRaw != null && limitRaw.toIntOrNull() == null) + ) { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid pagination")) + } + val page = pageRaw?.toInt() ?: 1 + val limit = limitRaw?.toInt() ?: 50 + if (page < 1 || limit !in 1..200) { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid pagination")) + } + call.respondNoStore(service.adminList(page, limit)) + } + } + put("/admin/rss/feeds/{id}/enabled") { + call.withRssAdmin(authService) { + val id = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id")) + val body = runCatching { call.receive() }.getOrElse { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + call.respondNoStore(service.adminSetEnabled(id, body.enabled)) + } + } + delete("/admin/rss/feeds/{id}") { + call.withRssAdmin(authService) { + val id = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id")) + service.adminDelete(id) + call.respond(HttpStatusCode.NoContent) + } + } + put("/admin/rss/users/{id}/enabled") { + call.withRssAdmin(authService) { + val userId = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing user id")) + val body = runCatching { call.receive() }.getOrElse { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + service.adminSetUserEnabled(userId, body.enabled) + call.respond(HttpStatusCode.NoContent) + } + } +} + +private suspend inline fun io.ktor.server.application.ApplicationCall.withRssAdmin( + authService: AuthService, + crossinline block: suspend () -> Unit, +) { + try { + withAdminAuth(authService) { block() } + } catch (error: RssFeedException) { + respondRssError(error) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt new file mode 100644 index 00000000..b36b0faa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt @@ -0,0 +1,141 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.RssFeedEnabledRequest +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReadResult +import dev.typetype.server.services.RssFeedReaderService +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.response.respondBytes +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +fun Route.rssFeedRoutes(service: RssFeedManagementService, authService: AuthService) { + get("/rss/feeds") { + call.withRssUser(authService) { userId -> call.respondNoStore(service.list(userId)) } + } + post("/rss/feeds") { + call.withRssUser(authService) { userId -> + val body = call.rssBody() ?: return@withRssUser + call.response.headers.append(HttpHeaders.CacheControl, "no-store") + call.respond(HttpStatusCode.Created, service.create(userId, body)) + } + } + put("/rss/feeds/{id}") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + val body = call.rssBody() ?: return@withRssUser + call.respondNoStore(service.update(userId, id, body)) + } + } + put("/rss/feeds/{id}/enabled") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + val body = call.rssBody() ?: return@withRssUser + call.respondNoStore(service.setEnabled(userId, id, body.enabled)) + } + } + post("/rss/feeds/{id}/regenerate") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + call.respondNoStore(service.regenerate(userId, id)) + } + } + delete("/rss/feeds/{id}") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + service.delete(userId, id) + call.respond(HttpStatusCode.NoContent) + } + } +} + +fun Route.rssPublicRoutes(service: RssFeedReaderService) { + get("/rss/feeds/{file}") { + val feedId = call.parameters["file"]?.takeIf { it.endsWith(".xml") }?.removeSuffix(".xml") + ?: return@get call.respond(HttpStatusCode.NotFound) + val secret = call.request.queryParameters["token"]?.takeIf(String::isNotBlank) + ?: return@get call.respond(HttpStatusCode.NotFound) + when (val result = service.read(feedId, secret)) { + RssFeedReadResult.NotFound -> call.respond(HttpStatusCode.NotFound) + is RssFeedReadResult.Throttled -> { + call.response.headers.append(HttpHeaders.RetryAfter, result.retryAfterSeconds.toString()) + call.respond(HttpStatusCode.TooManyRequests, ErrorResponse("Too many RSS requests", "rss_rate_limited")) + } + is RssFeedReadResult.Ready -> call.respondRss(result) + } + } +} + +internal suspend fun ApplicationCall.respondRssError(error: RssFeedException) { + val status = when (error.code) { + "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound + "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden + "rss_feed_limit_reached" -> HttpStatusCode.Conflict + else -> HttpStatusCode.BadRequest + } + respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) +} + +private suspend inline fun ApplicationCall.withRssUser( + authService: AuthService, + crossinline block: suspend (String) -> Unit, +) { + try { + withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth respond(HttpStatusCode.Forbidden, ErrorResponse("Guest users cannot manage RSS feeds")) + } + block(userId) + } + } catch (error: RssFeedException) { + respondRssError(error) + } +} + +private suspend inline fun ApplicationCall.rssBody(): T? = runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body", "rss_invalid_body")) + null +} + +private suspend fun ApplicationCall.rssFeedId(): String? = parameters["id"] ?: run { + respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id", "rss_missing_feed_id")) + null +} + +private suspend fun ApplicationCall.respondRss(result: RssFeedReadResult.Ready) { + response.headers.append(HttpHeaders.ETag, result.etag) + response.headers.append(HttpHeaders.LastModified, RFC_1123.format(Instant.ofEpochMilli(result.lastModified))) + response.headers.append(HttpHeaders.CacheControl, "private, max-age=${result.maxAgeSeconds}, must-revalidate") + val ifNoneMatch = request.headers[HttpHeaders.IfNoneMatch] + val unchanged = if (ifNoneMatch != null) { + etagMatches(ifNoneMatch, result.etag) + } else { + request.headers[HttpHeaders.IfModifiedSince]?.let(::parseHttpDate)?.let { since -> + !Instant.ofEpochMilli(result.lastModified).truncatedTo(ChronoUnit.SECONDS).isAfter(since) + } == true + } + if (unchanged) return respond(HttpStatusCode.NotModified) + respondBytes(result.bytes, ContentType.parse("application/rss+xml; charset=utf-8")) +} + +private fun parseHttpDate(value: String): Instant? = runCatching { Instant.from(RFC_1123.parse(value)) }.getOrNull() +private fun etagMatches(header: String, etag: String): Boolean = header.split(',').any { candidate -> + candidate.trim().let { it == "*" || it.removePrefix("W/") == etag.removePrefix("W/") } +} +private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index eab6f335..308248ac 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -20,6 +20,7 @@ internal fun Route.userDataRoutes( subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) subscriptionFeedRoutes(svc.subscriptionFeedService, authService) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) + rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) savedPlaylistRoutes(svc.savedPlaylistService, svc.publicPlaylistService, authService) watchLaterRoutes(svc.watchLaterService, authService, svc.videoMetadataRepairService) From 9ffa81525ec07682530a7976098a7fa655ad5bd5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:01 +0200 Subject: [PATCH 16/65] docs: document private RSS feed API --- openapi.yaml | 14 +++ openapi/components/access-control.yaml | 6 ++ openapi/components/instance.yaml | 12 +++ openapi/components/rss.yaml | 67 ++++++++++++++ openapi/paths/rss-admin.yaml | 69 ++++++++++++++ openapi/paths/rss.yaml | 121 +++++++++++++++++++++++++ 6 files changed, 289 insertions(+) create mode 100644 openapi/components/rss.yaml create mode 100644 openapi/paths/rss-admin.yaml create mode 100644 openapi/paths/rss.yaml diff --git a/openapi.yaml b/openapi.yaml index 00121b5c..f551cc64 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -16,6 +16,7 @@ tags: - name: downloader - name: youtube-session - name: user-data + - name: rss paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } @@ -43,6 +44,11 @@ paths: /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } + /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } + /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } + /rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss.yaml#/RssFeedEnabled } + /rss/feeds/{id}/regenerate: { $ref: ./openapi/paths/rss.yaml#/RssFeedRegenerate } + /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } @@ -51,6 +57,10 @@ paths: /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } /allowed/channels/{channelUrl}: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannel } /admin/settings: { $ref: ./openapi/paths/access-control.yaml#/AdminSettings } + /admin/rss/feeds: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeeds } + /admin/rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeedEnabled } + /admin/rss/feeds/{id}: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeed } + /admin/rss/users/{id}/enabled: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssUserEnabled } /admin/users: { $ref: ./openapi/paths/admin-users.yaml#/AdminUsers } /admin/users/{id}/access-mode: { $ref: ./openapi/paths/admin-users.yaml#/AdminUserAccessMode } /admin/users/managed-access: { $ref: ./openapi/paths/admin-managed-access.yaml#/AdminManagedAccessUsers } @@ -138,6 +148,10 @@ components: SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } + RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } + RssFeedItem: { $ref: ./openapi/components/rss.yaml#/RssFeedItem } + RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } + AdminRssFeedsPage: { $ref: ./openapi/components/rss.yaml#/AdminRssFeedsPage } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index f3e34bab..d8ad0e10 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -37,6 +37,12 @@ SettingsItem: type: string enum: [unrestricted, allow_list] default: unrestricted + rssEnabled: { type: boolean, default: false } + rssPublicBaseUrl: { type: string, format: uri, nullable: true } + rssMaxFeedsPerUser: { type: integer, minimum: 1, maximum: 100, default: 10 } + rssMaxItems: { type: integer, minimum: 1, maximum: 200, default: 50 } + rssMinimumPollMinutes: { type: integer, minimum: 1, maximum: 1440, default: 5 } + rssRateLimitPerMinute: { type: integer, minimum: 1, maximum: 600, default: 30 } AllowedChannelItem: type: object required: [url] diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 385bb0fc..a0629e47 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,6 +20,7 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady + - rss properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -49,3 +50,14 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] + rss: + $ref: '#/RssInstanceCapability' +RssInstanceCapability: + type: object + required: [enabled, maxFeedsPerUser, maxItems, minimumPollMinutes, rateLimitPerMinute] + properties: + enabled: { type: boolean } + maxFeedsPerUser: { type: integer } + maxItems: { type: integer } + minimumPollMinutes: { type: integer } + rateLimitPerMinute: { type: integer } diff --git a/openapi/components/rss.yaml b/openapi/components/rss.yaml new file mode 100644 index 00000000..cbd66a5d --- /dev/null +++ b/openapi/components/rss.yaml @@ -0,0 +1,67 @@ +RssFeedRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 100 } + scope: { type: string, enum: [all, channels], default: all } + channelUrls: + type: array + maxItems: 100 + items: { type: string, format: uri } + serviceIds: + type: array + minItems: 1 + uniqueItems: true + default: [0, 5, 6] + items: { type: integer, enum: [0, 5, 6] } + includeVideos: { type: boolean, default: true } + includeShorts: { type: boolean, default: true } + includeLive: { type: boolean, default: true } + includeUpcoming: { type: boolean, default: true } +RssFeedItem: + allOf: + - $ref: '#/RssFeedRequest' + - type: object + required: [id, enabled, createdAt, updatedAt, lastUsedAt] + properties: + id: { type: string } + enabled: { type: boolean } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } + lastUsedAt: { type: integer, format: int64, nullable: true } +RssFeedSecretItem: + type: object + required: [feed, feedUrl] + properties: + feed: { $ref: '#/RssFeedItem' } + feedUrl: + type: string + format: uri + description: Returned only after creation or secret regeneration. +RssFeedEnabledRequest: + type: object + required: [enabled] + properties: + enabled: { type: boolean } +RssUserPolicyRequest: + $ref: '#/RssFeedEnabledRequest' +AdminRssFeedItem: + type: object + required: [feed, userId, userName, userEmail, userRssEnabled, userSuspended] + properties: + feed: { $ref: '#/RssFeedItem' } + userId: { type: string } + userName: { type: string } + userEmail: { type: string, format: email } + userRssEnabled: { type: boolean } + userSuspended: { type: boolean } +AdminRssFeedsPage: + type: object + required: [items, page, limit, total] + properties: + items: + type: array + items: { $ref: '#/AdminRssFeedItem' } + page: { type: integer } + limit: { type: integer } + total: { type: integer, format: int64 } diff --git a/openapi/paths/rss-admin.yaml b/openapi/paths/rss-admin.yaml new file mode 100644 index 00000000..0c431b8f --- /dev/null +++ b/openapi/paths/rss-admin.yaml @@ -0,0 +1,69 @@ +AdminRssFeeds: + get: + tags: [rss] + summary: List private RSS feeds across the instance + security: [{ bearerAuth: [] }] + parameters: + - { name: page, in: query, schema: { type: integer, minimum: 1, default: 1 } } + - { name: limit, in: query, schema: { type: integer, minimum: 1, maximum: 200, default: 50 } } + responses: + '200': + description: Paginated RSS feeds and owners + content: + application/json: + schema: { $ref: ../components/rss.yaml#/AdminRssFeedsPage } + '400': { description: Invalid pagination } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } +AdminRssFeedEnabled: + put: + tags: [rss] + summary: Enable or disable one RSS feed as an admin + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedEnabledRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Feed not found } +AdminRssFeed: + delete: + tags: [rss] + summary: Revoke and delete one RSS feed as an admin + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '204': { description: Deleted } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Feed not found } +AdminRssUserEnabled: + put: + tags: [rss] + summary: Enable or disable RSS for one account + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssUserPolicyRequest } + responses: + '204': { description: Account RSS policy updated } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Account not found } diff --git a/openapi/paths/rss.yaml b/openapi/paths/rss.yaml new file mode 100644 index 00000000..bd8e610c --- /dev/null +++ b/openapi/paths/rss.yaml @@ -0,0 +1,121 @@ +RssFeeds: + get: + tags: [rss] + summary: List private RSS feeds for the current account + security: [{ bearerAuth: [] }] + responses: + '200': + description: RSS feeds without their secrets + content: + application/json: + schema: { type: array, items: { $ref: ../components/rss.yaml#/RssFeedItem } } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + post: + tags: [rss] + summary: Create a private RSS feed + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedRequest } + responses: + '201': + description: Created feed and its one-time private URL + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedSecretItem } + '400': { description: Invalid scope or filters } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '409': { description: Account feed limit reached } +RssFeed: + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + put: + tags: [rss] + summary: Replace a private RSS feed configuration + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid scope or filters } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } + delete: + tags: [rss] + summary: Delete a private RSS feed + security: [{ bearerAuth: [] }] + responses: + '204': { description: Deleted } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedEnabled: + put: + tags: [rss] + summary: Enable or disable a private RSS feed + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedEnabledRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedRegenerate: + post: + tags: [rss] + summary: Revoke the current secret and issue a new private URL + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '200': + description: Feed and its replacement one-time private URL + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedSecretItem } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedDocument: + get: + tags: [rss] + summary: Read a private RSS 2.0 document + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + - { name: token, in: query, required: true, schema: { type: string } } + responses: + '200': + description: RSS document + headers: + ETag: { schema: { type: string } } + Last-Modified: { schema: { type: string } } + Cache-Control: { schema: { type: string } } + content: + application/rss+xml: + schema: { type: string } + '304': { description: Document unchanged } + '404': { description: Invalid, disabled, or revoked feed } + '429': { description: Feed request limit exceeded } From 33717e21e4e49c0a9a0ccf7def858a0355279dd9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 17/65] test: cover RSS feed management --- .../server/RssFeedManagementServiceTest.kt | 155 ++++++++++++++++++ .../dev/typetype/server/RssFeedSecretTest.kt | 23 +++ 2 files changed, 178 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt new file mode 100644 index 00000000..b6f82cc4 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt @@ -0,0 +1,155 @@ +package dev.typetype.server + +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction + +class RssFeedManagementServiceTest { + private val settings = AdminSettingsService() + private val subscriptions = SubscriptionsService() + private val service = RssFeedManagementService(settings, subscriptions) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + } + + @Test + fun `selected channels must be owned subscriptions and feeds stay isolated`() = runTest { + enableRss() + insertUser("user-a") + insertUser("user-b") + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + subscriptions.add("user-b", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@b", "B")) + + val created = service.create( + "user-a", + RssFeedRequest(name = "A only", scope = "channels", channelUrls = listOf("https://youtube.com/@a/")), + ) + + assertTrue(created.feedUrl.startsWith("https://video.example/api/rss/feeds/")) + assertEquals(listOf("https://youtube.com/@a"), created.feed.channelUrls) + assertEquals(listOf(created.feed), service.list("user-a")) + assertTrue(service.list("user-b").isEmpty()) + val error = runCatching { + service.create( + "user-b", + RssFeedRequest(name = "Not mine", scope = "channels", channelUrls = listOf("https://youtube.com/@a")), + ) + }.exceptionOrNull() as RssFeedException + assertEquals("rss_channel_not_subscribed", error.code) + } + + @Test + fun `accounts cannot mutate feeds they do not own`() = runTest { + enableRss() + insertUser("user-a") + insertUser("user-b") + val created = service.create("user-a", RssFeedRequest(name = "Private feed")) + + val attempts = listOf Unit>( + { service.update("user-b", created.feed.id, RssFeedRequest(name = "Changed")) }, + { service.setEnabled("user-b", created.feed.id, false) }, + { service.regenerate("user-b", created.feed.id) }, + { service.delete("user-b", created.feed.id) }, + ) + + attempts.forEach { attempt -> + val error = runCatching { attempt() }.exceptionOrNull() as RssFeedException + assertEquals("rss_feed_not_found", error.code) + } + assertEquals(listOf(created.feed), service.list("user-a")) + } + + @Test + fun `disabled account retains feeds but cannot manage them`() = runTest { + enableRss() + insertUser("user-a") + service.create("user-a", RssFeedRequest(name = "All")) + service.adminSetUserEnabled("user-a", false) + + val error = runCatching { service.list("user-a") }.exceptionOrNull() as RssFeedException + assertEquals("rss_user_disabled", error.code) + assertEquals(1L, service.adminList(1, 20).total) + } + + @Test + fun `global disable retains feed configuration`() = runTest { + enableRss() + insertUser("user-a") + val created = service.create("user-a", RssFeedRequest(name = "All")) + settings.upsert(AdminSettingsItem(rssEnabled = false, rssPublicBaseUrl = "https://video.example")) + + val error = runCatching { service.list("user-a") }.exceptionOrNull() as RssFeedException + assertEquals("rss_disabled", error.code) + assertEquals(created.feed, service.adminList(1, 20).items.single().feed) + + enableRss() + assertEquals(listOf(created.feed), service.list("user-a")) + } + + @Test + fun `admin cannot create an RSS policy for an unknown account`() = runTest { + val error = runCatching { service.adminSetUserEnabled("missing", false) } + .exceptionOrNull() as RssFeedException + + assertEquals("rss_user_not_found", error.code) + } + + @Test + fun `concurrent creation cannot exceed the account feed limit`() = runTest { + settings.upsert( + AdminSettingsItem( + rssEnabled = true, + rssPublicBaseUrl = "https://video.example", + rssMaxFeedsPerUser = 1, + ), + ) + insertUser("user-a") + + val attempts = List(4) { index -> + async { runCatching { service.create("user-a", RssFeedRequest(name = "Feed $index")) } } + }.awaitAll() + + assertEquals(1, attempts.count(Result<*>::isSuccess)) + assertTrue(attempts.filter(Result<*>::isFailure).all { + (it.exceptionOrNull() as RssFeedException).code == "rss_feed_limit_reached" + }) + } + + private suspend fun enableRss() { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + } + + private fun insertUser(id: String) = transaction { + UsersTable.insert { + it[UsersTable.id] = id + it[email] = "$id@test.local" + it[passwordHash] = "hash" + it[name] = id + it[role] = "user" + it[createdAt] = 1L + it[updatedAt] = 1L + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt new file mode 100644 index 00000000..c85db17e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt @@ -0,0 +1,23 @@ +package dev.typetype.server + +import dev.typetype.server.services.RssFeedSecret +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssFeedSecretTest { + private val secrets = RssFeedSecret() + + @Test + fun `secret is random and only its hash is comparable`() { + val first = secrets.create() + val second = secrets.create() + val hash = secrets.hash(first) + + assertNotEquals(first, second) + assertNotEquals(first, hash) + assertTrue(secrets.matches(first, hash)) + assertFalse(secrets.matches(second, hash)) + } +} From 7a520f6b62bd04f363b1d2af5c786a3afee59107 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 18/65] test: cover private RSS delivery --- .../server/RssFeedReaderRoutesTest.kt | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt new file mode 100644 index 00000000..2c109240 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt @@ -0,0 +1,221 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.routes.rssPublicRoutes +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.BlockedService +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReaderService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import java.net.URI +import java.net.URLEncoder +import java.nio.charset.StandardCharsets + +class RssFeedReaderRoutesTest { + private val settings = AdminSettingsService() + private val subscriptions = SubscriptionsService() + private val feed = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + private val blocked = BlockedService() + private val management = RssFeedManagementService(settings, subscriptions) + private val reader = RssFeedReaderService(settings, feed, blocked) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + insertUser() + } + + @Test + fun `private feed supports conditional cache and immediate secret revocation`() = testApplication { + val created = createFeed() + val uri = URI(created.feedUrl) + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + val first = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + assertEquals(HttpStatusCode.OK, first.status) + assertEquals("application/rss+xml; charset=utf-8", first.headers[HttpHeaders.ContentType]) + assertEquals("private, max-age=300, must-revalidate", first.headers[HttpHeaders.CacheControl]) + assertTrue(first.bodyAsText().contains("")) + assertTrue(first.bodyAsText().contains("https://video.example/watch?v=")) + val etag = first.headers[HttpHeaders.ETag]!! + val lastModified = first.headers[HttpHeaders.LastModified]!! + + val cached = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) { + headers.append(HttpHeaders.IfNoneMatch, "W/$etag") + } + assertEquals(HttpStatusCode.NotModified, cached.status) + + val changed = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) { + headers.append(HttpHeaders.IfNoneMatch, "\"different\"") + headers.append(HttpHeaders.IfModifiedSince, lastModified) + } + assertEquals(HttpStatusCode.OK, changed.status) + + val regenerated = management.regenerate("user-a", created.feed.id) + assertEquals(HttpStatusCode.NotFound, client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery).status) + assertFalse(regenerated.feedUrl.endsWith(uri.rawQuery)) + } + + @Test + fun `blocked videos channels and keywords are excluded from RSS output`() = testApplication { + val created = createFeed() + val uri = URI(created.feedUrl) + val path = uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + blocked.addVideo("user-a", "https://youtube.com/@a/video") + assertTrue(blocked.profileFor("user-a").blocksVideo("https://youtube.com/@a/video")) + val blockedVideoUrl = URLEncoder.encode("https://youtube.com/@a/video", StandardCharsets.UTF_8) + assertFalse(client.get(path).bodyAsText().contains(blockedVideoUrl)) + blocked.deleteVideo("user-a", "https://youtube.com/@a/video", "user") + + blocked.addChannel("user-a", "https://youtube.com/@a") + assertTrue(blocked.profileFor("user-a").blocksChannel("https://youtube.com/@a", "A")) + assertFalse(client.get(path).bodyAsText().contains("")) + blocked.deleteChannel("user-a", "https://youtube.com/@a", "user") + + blocked.addKeyword("user-a", "video") + assertFalse(client.get(path).bodyAsText().contains("")) + } + + @Test + fun `RSS reads an existing snapshot without starting extraction`() = testApplication { + val channel = CountingChannelService() + val snapshotOnlyFeed = SubscriptionFeedService(subscriptions, channel, FakeCacheService()) + val snapshotOnlyReader = RssFeedReaderService(settings, snapshotOnlyFeed, blocked) + val created = createUnprimedFeed() + val uri = URI(created.feedUrl) + application { routing { rssPublicRoutes(snapshotOnlyReader) } } + + val response = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + + assertEquals(HttpStatusCode.OK, response.status) + assertFalse(response.bodyAsText().contains("")) + assertEquals(0, channel.calls) + } + + @Test + fun `RSS enforces configured item and request limits`() = testApplication { + settings.upsert( + AdminSettingsItem( + rssEnabled = true, + rssPublicBaseUrl = "https://video.example", + rssMaxItems = 1, + rssRateLimitPerMinute = 1, + ), + ) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@b", "B")) + val created = management.create("user-a", RssFeedRequest(name = "Limited feed")) + feed.getAll("user-a") + feed.awaitRefresh("user-a") + val uri = URI(created.feedUrl) + val path = uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + val first = client.get(path) + assertEquals(1, "".toRegex().findAll(first.bodyAsText()).count()) + val throttled = client.get(path) + assertEquals(HttpStatusCode.TooManyRequests, throttled.status) + assertTrue(throttled.headers[HttpHeaders.RetryAfter]?.toIntOrNull() in 1..60) + } + + @Test + fun `RSS rejects feeds owned by suspended accounts`() = testApplication { + val created = createFeed() + transaction { + UsersTable.update({ UsersTable.id eq "user-a" }) { it[suspended] = true } + } + val uri = URI(created.feedUrl) + application { routing { rssPublicRoutes(reader) } } + + val response = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + + assertEquals(HttpStatusCode.NotFound, response.status) + } + + private suspend fun createFeed(): dev.typetype.server.models.RssFeedSecretItem { + val created = createUnprimedFeed() + feed.getAll("user-a") + feed.awaitRefresh("user-a") + return created + } + + private suspend fun createUnprimedFeed(): dev.typetype.server.models.RssFeedSecretItem { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + return management.create("user-a", RssFeedRequest(name = "My feed")) + } + + private fun insertUser() = transaction { + UsersTable.insert { + it[id] = "user-a" + it[email] = "rss-reader@test.local" + it[passwordHash] = "hash" + it[name] = "RSS reader" + it[role] = "user" + it[createdAt] = 1L + it[updatedAt] = 1L + } + } + + private class CountingChannelService : ChannelService { + var calls = 0 + + override suspend fun getChannel( + url: String, + nextpage: String?, + sort: String?, + ): ExtractionResult { + calls += 1 + return ExtractionResult.Success(ChannelResponse("", "", "", "", 0, false, emptyList(), null)) + } + + override suspend fun getPlaylists( + url: String, + nextpage: String?, + ): ExtractionResult = + ExtractionResult.Success(ChannelPlaylistsResponse(emptyList(), null)) + } +} From 50e41619e7803c20566744546c047f8eadcc13af Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 19/65] test: cover RSS policy and content filters --- .../dev/typetype/server/AdminRssRoutesTest.kt | 150 ++++++++++++++++++ .../typetype/server/RssVideoTypeFilterTest.kt | 118 ++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt new file mode 100644 index 00000000..600feb54 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt @@ -0,0 +1,150 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.routes.adminRssRoutes +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import org.jetbrains.exposed.v1.core.eq +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AdminRssRoutesTest { + private val settings = AdminSettingsService() + private val service = RssFeedManagementService(settings, SubscriptionsService()) + private val auth = AuthService.fixed(ADMIN_ID) + + companion object { + private const val ADMIN_ID = "rss-admin" + private const val USER_ID = "rss-user" + + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + insertUser(ADMIN_ID, "admin") + insertUser(USER_ID, "user") + } + + @Test + fun `admin can inspect and revoke a feed without receiving its secret`() = withApp { + enableRss() + val created = service.create(USER_ID, RssFeedRequest(name = "Private feed")) + + val listed = client.get("/admin/rss/feeds") { authorize() } + assertEquals(HttpStatusCode.OK, listed.status) + val body = listed.bodyAsText() + assertTrue(body.contains("user@test.local")) + assertTrue(body.contains(created.feed.id)) + assertFalse(body.contains("feedUrl")) + assertFalse(body.contains("token")) + + val deleted = client.delete("/admin/rss/feeds/${created.feed.id}") { authorize() } + assertEquals(HttpStatusCode.NoContent, deleted.status) + assertEquals(0L, service.adminList(1, 20).total) + } + + @Test + fun `admin account policy is retained and unknown accounts return 404`() = withApp { + enableRss() + service.create(USER_ID, RssFeedRequest(name = "Private feed")) + + val disabled = client.put("/admin/rss/users/$USER_ID/enabled") { + authorize() + contentType(ContentType.Application.Json) + setBody("""{"enabled":false}""") + } + assertEquals(HttpStatusCode.NoContent, disabled.status) + val item = service.adminList(1, 20).items.single() + assertFalse(item.userRssEnabled) + assertTrue(item.feed.enabled) + + val missing = client.put("/admin/rss/users/missing/enabled") { + authorize() + contentType(ContentType.Application.Json) + setBody("""{"enabled":false}""") + } + assertEquals(HttpStatusCode.NotFound, missing.status) + assertTrue(missing.bodyAsText().contains("rss_user_not_found")) + } + + @Test + fun `admin inventory reports suspended owners`() = withApp { + enableRss() + service.create(USER_ID, RssFeedRequest(name = "Private feed")) + transaction { + UsersTable.update({ UsersTable.id eq USER_ID }) { it[suspended] = true } + } + + val response = client.get("/admin/rss/feeds") { authorize() } + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().contains("\"userSuspended\":true")) + } + + @Test + fun `admin inventory rejects malformed pagination`() = withApp { + val response = client.get("/admin/rss/feeds?page=invalid") { authorize() } + + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json(Json { encodeDefaults = true }) } + routing { adminRssRoutes(service, auth) } + } + block() + } + + private suspend fun enableRss() { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun insertUser(id: String, role: String) = transaction { + UsersTable.insert { + it[UsersTable.id] = id + it[email] = role + "@test.local" + it[passwordHash] = "hash" + it[name] = role + it[UsersTable.role] = role + it[createdAt] = 1L + it[updatedAt] = 1L + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt b/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt new file mode 100644 index 00000000..b14e089b --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt @@ -0,0 +1,118 @@ +package dev.typetype.server + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem +import dev.typetype.server.services.RssVideoMetadata +import dev.typetype.server.services.RssVideoTypeFilter +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssVideoTypeFilterTest { + @Test + fun `separates regular shorts live and upcoming content`() { + assertTrue(RssVideoTypeFilter.includes(feed(includeVideos = true), video(), NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeVideos = false), video(), NOW)) + assertTrue(RssVideoTypeFilter.includes(feed(includeShorts = true), video(short = true), NOW)) + assertTrue(RssVideoTypeFilter.includes(feed(includeLive = true), video(live = true), NOW)) + assertTrue( + RssVideoTypeFilter.includes( + feed(includeUpcoming = true), + video(duration = -1, publishedAt = NOW + 60_000), + NOW, + ), + ) + } + + @Test + fun `does not treat past unknown-duration or post-live videos as upcoming`() { + val upcomingOnly = feed(includeUpcoming = true) + assertFalse( + RssVideoTypeFilter.includes( + upcomingOnly, + video(duration = -1, publishedAt = NOW - 60_000), + NOW, + ), + ) + assertFalse( + RssVideoTypeFilter.includes( + upcomingOnly, + video(duration = -1, publishedAt = NOW + 60_000, postLive = true), + NOW, + ), + ) + } + + @Test + fun `live and upcoming state takes priority over short form`() { + val liveShort = video(short = true, live = true) + assertTrue(RssVideoTypeFilter.includes(feed(includeLive = true), liveShort, NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeShorts = true), liveShort, NOW)) + + val upcomingShort = video(short = true, duration = -1, publishedAt = NOW + 60_000) + assertTrue(RssVideoTypeFilter.includes(feed(includeUpcoming = true), upcomingShort, NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeShorts = true), upcomingShort, NOW)) + } + + @Test + fun `detects supported provider from canonical and short hosts`() { + assertEquals(0, RssVideoMetadata.serviceId(video(url = "https://www.youtube.com/watch?v=video"))) + assertEquals(5, RssVideoMetadata.serviceId(video(url = "https://b23.tv/video"))) + assertEquals(5, RssVideoMetadata.serviceId(video(url = "https://www.bilibili.com/video/BV1"))) + assertEquals(6, RssVideoMetadata.serviceId(video(url = "https://nico.ms/sm1"))) + assertEquals(6, RssVideoMetadata.serviceId(video(url = "https://www.nicovideo.jp/watch/sm1"))) + } + + private fun feed( + includeVideos: Boolean = false, + includeShorts: Boolean = false, + includeLive: Boolean = false, + includeUpcoming: Boolean = false, + ) = RssFeedItem( + id = "feed", + name = "Feed", + scope = "all", + channelUrls = emptyList(), + serviceIds = listOf(0), + includeVideos = includeVideos, + includeShorts = includeShorts, + includeLive = includeLive, + includeUpcoming = includeUpcoming, + enabled = true, + createdAt = NOW, + updatedAt = NOW, + ) + + private fun video( + duration: Long = 120, + publishedAt: Long = NOW - 60_000, + short: Boolean = false, + live: Boolean = false, + postLive: Boolean = false, + url: String = "https://youtube.com/watch?v=video", + ) = VideoItem( + id = "video", + title = "Video", + url = url, + thumbnailUrl = "", + uploaderName = "Channel", + uploaderUrl = "https://youtube.com/@channel", + uploaderAvatarUrl = "", + duration = duration, + viewCount = 0, + uploadDate = "", + streamType = "video_stream", + isShortFormContent = short, + uploaderVerified = false, + shortDescription = null, + publishedAt = publishedAt, + isLive = live, + isPostLive = postLive, + isLiveContent = live || postLive, + ) + + private companion object { + const val NOW = 1_800_000_000_000L + } +} From abed280dae88af084bd664a6b247a5e42d30e39b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 19:23:15 +0200 Subject: [PATCH 20/65] feat: hide live streams from subscription feeds --- openapi/components/access-control.yaml | 1 + .../server/db/SettingsSchemaMigrations.kt | 1 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../server/routes/SubscriptionFeedRoutes.kt | 10 +- .../typetype/server/routes/UserDataRoutes.kt | 2 +- .../server/services/RssVideoTypeFilter.kt | 5 +- .../services/SettingsPersistenceMappers.kt | 2 + .../server/services/SettingsService.kt | 5 + .../services/SubscriptionFeedService.kt | 8 +- .../services/SubscriptionFeedSnapshot.kt | 24 ++-- .../server/services/VideoItemSchedule.kt | 8 ++ .../SettingsPrivacyControlsRoutesTest.kt | 4 +- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 131 ++++++++++++++++++ 14 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index d8ad0e10..900de1fe 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -33,6 +33,7 @@ SettingsItem: hideRelatedVideos: { type: boolean, default: false } hideComments: { type: boolean, default: false } hideShorts: { type: boolean, default: false } + hideSubscriptionLiveStreams: { type: boolean, default: false } accessMode: type: string enum: [unrestricted, allow_list] diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index cb816f2f..45566d0f 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -24,6 +24,7 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_related_videos BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_comments BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_shorts BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_subscription_live_streams BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index d4b78c50..4744441f 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -32,6 +32,7 @@ object SettingsTable : Table("settings") { val hideRelatedVideos = bool("hide_related_videos").default(false) val hideComments = bool("hide_comments").default(false) val hideShorts = bool("hide_shorts").default(false) + val hideSubscriptionLiveStreams = bool("hide_subscription_live_streams").default(false) val disableWatchHistory = bool("disable_watch_history").default(false) val deArrowEnabled = bool("dearrow_enabled").default(false) val deArrowTitleMode = text("dearrow_title_mode").default("dearrow") diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index e471b347..b6f46a83 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -32,6 +32,7 @@ data class SettingsItem( val hideRelatedVideos: Boolean = false, val hideComments: Boolean = false, val hideShorts: Boolean = false, + val hideSubscriptionLiveStreams: Boolean = false, val disableWatchHistory: Boolean = false, val deArrowEnabled: Boolean = false, val deArrowTitleMode: String = "dearrow", diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 110d257c..07154402 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.SubscriptionFeedPreparingResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SettingsService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond @@ -13,14 +14,19 @@ import io.ktor.server.routing.get private const val MAX_FEED_PAGE = 10_000 -fun Route.subscriptionFeedRoutes(feedService: SubscriptionFeedService, authService: AuthService) { +fun Route.subscriptionFeedRoutes( + feedService: SubscriptionFeedService, + authService: AuthService, + settingsService: SettingsService? = null, +) { get("/subscriptions/feed") { call.withJwtAuth(authService) { userId -> val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 val cursor = call.request.queryParameters["cursor"] + val hideLiveStreams = settingsService?.hidesSubscriptionLiveStreams(userId) ?: false call.response.headers.append(HttpHeaders.CacheControl, "no-store") - when (val result = feedService.getPage(userId, page, limit, cursor)) { + when (val result = feedService.getPage(userId, page, limit, cursor, hideLiveStreams)) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) is SubscriptionFeedPageResult.Preparing -> { call.response.headers.append(HttpHeaders.RetryAfter, "1") diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 308248ac..74baab91 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -18,7 +18,7 @@ internal fun Route.userDataRoutes( ) { historyRoutes(svc.historyService, authService, svc.settingsService) subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) - subscriptionFeedRoutes(svc.subscriptionFeedService, authService) + subscriptionFeedRoutes(svc.subscriptionFeedService, authService, svc.settingsService) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt index 1e4e3dc6..a7919c07 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -6,11 +6,8 @@ import dev.typetype.server.models.VideoItem internal object RssVideoTypeFilter { fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { video.isLive -> feed.includeLive - isUpcoming(video, now) -> feed.includeUpcoming + video.isUpcomingAt(now) -> feed.includeUpcoming video.isShortFormContent -> feed.includeShorts else -> feed.includeVideos } - - private fun isUpcoming(video: VideoItem, now: Long): Boolean = - !video.isPostLive && video.duration < 0 && RssVideoMetadata.publishedAtMillis(video) > now } diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 451019b7..f40a0237 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -45,6 +45,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( hideRelatedVideos = this[SettingsTable.hideRelatedVideos], hideComments = this[SettingsTable.hideComments], hideShorts = this[SettingsTable.hideShorts], + hideSubscriptionLiveStreams = this[SettingsTable.hideSubscriptionLiveStreams], disableWatchHistory = this[SettingsTable.disableWatchHistory], deArrowEnabled = this[SettingsTable.deArrowEnabled], deArrowTitleMode = this[SettingsTable.deArrowTitleMode], @@ -82,6 +83,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.hideRelatedVideos] = settings.hideRelatedVideos this[SettingsTable.hideComments] = settings.hideComments this[SettingsTable.hideShorts] = settings.hideShorts + this[SettingsTable.hideSubscriptionLiveStreams] = settings.hideSubscriptionLiveStreams this[SettingsTable.disableWatchHistory] = settings.disableWatchHistory this[SettingsTable.deArrowEnabled] = settings.deArrowEnabled this[SettingsTable.deArrowTitleMode] = settings.deArrowTitleMode diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index 2e73bbf3..1dffe3c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,6 +35,11 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } + suspend fun hidesSubscriptionLiveStreams(userId: String): Boolean = DatabaseFactory.query { + SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull() + ?.get(SettingsTable.hideSubscriptionLiveStreams) ?: false + } + suspend fun getAccessModePolicy(userId: String): AccessModePolicy = DatabaseFactory.query { SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { AccessModePolicy( diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 602c634e..f641e2ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -36,6 +36,7 @@ class SubscriptionFeedService( page: Int, limit: Int, cursor: String?, + hideLiveStreams: Boolean = false, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -49,6 +50,9 @@ class SubscriptionFeedService( val cursorState = cursor?.let(SubscriptionFeedCursorCodec::decode) if (cursor != null && cursorState == null) return SubscriptionFeedPageResult.InvalidCursor if (cursorState != null && cursorState.limit != limit) return SubscriptionFeedPageResult.InvalidCursor + if (cursorState != null && cursorState.hideLiveStreams != hideLiveStreams) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current @@ -56,7 +60,9 @@ class SubscriptionFeedService( ?: return SubscriptionFeedPageResult.StaleGeneration } val offset = cursorState?.offset ?: page * limit - return SubscriptionFeedPageResult.Ready(snapshot.page(offset, limit, isRefreshing(userId))) + return SubscriptionFeedPageResult.Ready( + snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams), + ) } suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index b48aa4a8..2f7fd2ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -20,13 +20,14 @@ private data class SubscriptionFeedCursor( val generation: Long, val offset: Int, val limit: Int, + val hideLiveStreams: Boolean = false, ) internal object SubscriptionFeedCursorCodec { - fun encode(generation: Long, offset: Int, limit: Int): String { + fun encode(generation: Long, offset: Int, limit: Int, hideLiveStreams: Boolean): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit), + SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -35,7 +36,7 @@ internal object SubscriptionFeedCursorCodec { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } - ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit) } + ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit, it.hideLiveStreams) } }.getOrNull() } @@ -43,22 +44,29 @@ internal data class SubscriptionFeedCursorState( val generation: Long, val offset: Int, val limit: Int, + val hideLiveStreams: Boolean, ) internal fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, + hideLiveStreams: Boolean = false, ): SubscriptionFeedResponse { - val from = offset.coerceAtMost(videos.size) - val to = minOf(from + limit, videos.size) - val nextpage = if (to < videos.size) { - SubscriptionFeedCursorCodec.encode(generation, to, limit) + val visibleVideos = if (hideLiveStreams) { + videos.filterNot { it.isLiveOrUpcomingAt(generatedAt) } + } else { + videos + } + val from = offset.coerceAtMost(visibleVideos.size) + val to = minOf(from + limit, visibleVideos.size) + val nextpage = if (to < visibleVideos.size) { + SubscriptionFeedCursorCodec.encode(generation, to, limit, hideLiveStreams) } else { null } return SubscriptionFeedResponse( - videos = videos.subList(from, to), + videos = visibleVideos.subList(from, to), nextpage = nextpage, generation = generation, generatedAt = generatedAt, diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt new file mode 100644 index 00000000..977b3cd9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -0,0 +1,8 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +internal fun VideoItem.isUpcomingAt(now: Long): Boolean = + !isPostLive && duration < 0 && RssVideoMetadata.publishedAtMillis(this) > now + +internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) diff --git a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt index 04ca28e9..cf6891bd 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt @@ -63,6 +63,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideRelatedVideos\":false", "\"hideComments\":false", "\"hideShorts\":false", + "\"hideSubscriptionLiveStreams\":false", ), ) } @@ -87,6 +88,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideRelatedVideos\":true", "\"hideComments\":true", "\"hideShorts\":true", + "\"hideSubscriptionLiveStreams\":true", ), ) } @@ -106,6 +108,6 @@ class SettingsPrivacyControlsRoutesTest { values.forEach { assertTrue(body.contains(it)) } private fun settingsBody(sponsorBlockMode: String = "mark_only"): String = """ - {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true} + {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true} """.trimIndent() } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt new file mode 100644 index 00000000..c92392f2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -0,0 +1,131 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.channel +import dev.typetype.server.SubscriptionFeedTestFixtures.subscription +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.routes.subscriptionFeedRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionFeedLiveVisibilityRoutesTest { + private val auth = AuthService.fixed(TEST_USER_ID) + private val settingsService = SettingsService() + private lateinit var subscriptionsService: SubscriptionsService + private lateinit var feedService: SubscriptionFeedService + + companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + subscriptionsService = SubscriptionsService() + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( + duration = -1L, + publishedAt = System.currentTimeMillis() + 86_400_000L, + ), + video(3_000L, url = "https://youtube.com/watch?v=normal-1"), + video(2_000L, url = "https://youtube.com/watch?v=normal-2"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + } + + @Test + fun `account setting hides live streams before pagination`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val first = readPage(requestFeed(limit = 1)) + val cursor = requireNotNull(first.nextpage) + val second = readPage(requestFeed(limit = 1, cursor = cursor)) + + assertEquals(listOf("normal-1"), first.videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("normal-2"), second.videos.map { it.url.substringAfter("v=") }) + assertTrue(second.nextpage == null) + } + + @Test + fun `cursor is rejected after live visibility changes`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(readPage(requestFeed(limit = 1)).nextpage) + + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.BadRequest, requestFeed(limit = 1, cursor = cursor).status) + } + + @Test + fun `hidden live streams do not remove finished recordings`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_000L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + isLiveContent = true, + ), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("replay"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionFeedRoutes(feedService, auth, settingsService) } + } + block() + } + + private suspend fun ApplicationTestBuilder.requestFeed( + limit: Int, + cursor: String? = null, + ): HttpResponse = client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + } + + private suspend fun readPage(response: HttpResponse): SubscriptionFeedResponse { + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } +} From 815aec522fcdfd236e36a386b23dd26011f66de4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 10 Aug 2026 10:36:30 +0200 Subject: [PATCH 21/65] fix: drop stale live videos from feeds --- .../services/HomeRecommendationPoolBuilder.kt | 4 +++- .../services/HomeRecommendationPoolCache.kt | 2 +- .../server/services/SubscriptionFeedBuilder.kt | 11 +++++++++-- .../server/HomeRecommendationPoolBuilderTest.kt | 16 ++++++++++++++++ .../server/SubscriptionFeedRoutesTest.kt | 16 ++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt index 2fb091b7..489a596d 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt @@ -56,7 +56,9 @@ class HomeRecommendationPoolBuilder { if (video.url in profile.feedbackBlockedVideos || video.url in profile.implicitBlockedVideos) return@forEach if (video.uploaderUrl.isNotBlank() && video.uploaderUrl in profile.blockedChannels) return@forEach if (video.uploaderUrl.isNotBlank() && video.uploaderUrl in profile.feedbackBlockedChannels) return@forEach - if (!allowLive && HomeRecommendationLiveTitleDetector.isLiveLike(video.title)) return@forEach + if (!allowLive && (video.isLive || HomeRecommendationLiveTitleDetector.isLiveLike(video.title))) { + return@forEach + } val score = scorer(video, profile) val scored = HomeRecommendationScoredVideo(video = video, score = score, source = tagged.source) val current = byUrl[video.url] diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt index 229de660..17525b84 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt @@ -41,6 +41,6 @@ class HomeRecommendationPoolCache(private val cache: dev.typetype.server.cache.C companion object { private const val CACHE_TTL_SECONDS = 3_600L private const val STALE_TTL_SECONDS = 86_400L - private const val CACHE_VERSION = 8 + private const val CACHE_VERSION = 9 } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index 16d87f35..bae327dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -37,9 +37,16 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private suspend fun fetchSubscription(channelUrl: String): SubscriptionSourceResult = coroutineScope { val channel = async { fetchVideos(channelUrl) } val live = if (isYoutubeUrl(channelUrl)) async { fetchVideos(channelUrl.toLivestreamsTabUrl()) } else null - val results = listOfNotNull(channel.await(), live?.await()) + val channelResult = channel.await() + val liveResult = live?.await() + val videos = if (liveResult == null) { + channelResult.videos + } else { + channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos + } + val results = listOfNotNull(channelResult, liveResult) SubscriptionSourceResult( - videos = mergeVideos(results.flatMap { it.videos }), + videos = mergeVideos(videos), successfulSources = results.count { it.success }, failedSources = results.count { !it.success }, ) diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt index d5da8d72..eea6d9f3 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt @@ -51,6 +51,22 @@ class HomeRecommendationPoolBuilderTest { assertTrue(pool.discovery.first().url.endsWith("/normal1")) } + @Test + fun `pool builder drops candidates marked live without relying on their title`() { + val profile = profile() + val discovery = listOf( + tagged( + video("live", "a", title = "Weekly tech roundup").copy(isLive = true), + HomeRecommendationSourceTag.DISCOVERY_THEME, + ), + tagged(video("normal", "b", title = "Weekly tech roundup"), HomeRecommendationSourceTag.DISCOVERY_THEME), + ) + + val pool = HomeRecommendationPoolBuilder().build(profile, emptyList(), discovery, context) + + assertEquals(listOf("https://yt.com/v/normal"), pool.discovery.map { it.url }) + } + @Test fun `pool builder excludes titles containing a blocked keyword`() { val profile = profile(blockedKeywords = setOf("SPONSORED")) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt index 1bcddd7a..426e74ff 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt @@ -121,6 +121,22 @@ class SubscriptionFeedRoutesTest { assertNotNull(feed.generatedAt) } + @Test + fun `stale live from channel page is absent when streams tab no longer lists it`() = withApp { + val channelUrl = "https://www.youtube.com/channel/UC1" + val staleLiveUrl = "https://www.youtube.com/watch?v=private" + subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) + coEvery { channelService.getChannel(channelUrl, null) } returns channel( + video(-1L, url = staleLiveUrl, live = true), + video(3000L, url = "https://www.youtube.com/watch?v=normal"), + ) + coEvery { channelService.getChannel("$channelUrl/streams", null) } returns channel() + + val feed = buildAndRead() + + assertEquals(listOf("https://www.youtube.com/watch?v=normal"), feed.videos.map { it.url }) + } + @Test fun `cursor keeps pagination on one generation after refresh`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) From eb47a56dd5909dde3528e38a4682cea31fe32122 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 11 Aug 2026 12:03:14 +0200 Subject: [PATCH 22/65] chore: align development version with 1.5.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 99db84dc..3f348d8d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.4.0 +appVersion=1.5.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From d3a20810b0cd80eda52bbb14093c4045dbcd712f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 12 Aug 2026 16:42:24 +0200 Subject: [PATCH 23/65] fix: proxy marked artifact redirects --- .../server/routes/DownloaderGatewayRoutes.kt | 7 +++-- .../DownloaderGatewayArtifactProxyTest.kt | 27 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt index 18ca2142..44b4e912 100644 --- a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt @@ -102,13 +102,16 @@ private fun shouldProxyArtifact(path: String, response: dev.typetype.server.serv if (!path.endsWith("/artifact")) return false if (response.status != 302 && response.status != 307) return false val location = headerValue(response, "Location") ?: return false - return isInternalHost(location) + val markedInternal = headerValue(response, INTERNAL_ARTIFACT_PROXY_HEADER) == "1" + return markedInternal || isLegacyInternalHost(location) } private fun headerValue(response: dev.typetype.server.services.DownloaderGatewayResponse, name: String): String? = response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second -private fun isInternalHost(location: String): Boolean { +private fun isLegacyInternalHost(location: String): Boolean { val host = runCatching { URI(location).host }.getOrNull() ?: return false return host.equals("garage", ignoreCase = true) } + +private const val INTERNAL_ARTIFACT_PROXY_HEADER = "X-TypeType-Artifact-Proxy" diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt index 292a8aec..0f4c544d 100644 --- a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt @@ -25,7 +25,8 @@ class DownloaderGatewayArtifactProxyTest { val requestedRange = AtomicReference() val upstream = HttpServer.create(InetSocketAddress(0), 0) upstream.createContext("/jobs/test/artifact") { exchange -> - exchange.responseHeaders.add(HttpHeaders.Location, "http://garage:${upstream.address.port}/object") + exchange.responseHeaders.add(HttpHeaders.Location, "http://typetype-garage:${upstream.address.port}/object") + exchange.responseHeaders.add("X-TypeType-Artifact-Proxy", "1") exchange.sendResponseHeaders(302, -1) exchange.close() } @@ -68,7 +69,29 @@ class DownloaderGatewayArtifactProxyTest { } } + @Test + fun `public artifact redirect remains external`() = testApplication { + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + exchange.responseHeaders.add(HttpHeaders.Location, "https://downloads.example.com/object") + exchange.sendResponseHeaders(302, -1) + exchange.close() + } + upstream.start() + val gateway = DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}") + application { routing { downloaderGatewayRoutes(gateway) } } + val noRedirectClient = createClient { followRedirects = false } + + try { + val response = noRedirectClient.get("/downloader/jobs/test/artifact") + assertEquals(HttpStatusCode.Found, response.status) + assertEquals("https://downloads.example.com/object", response.headers[HttpHeaders.Location]) + } finally { + upstream.stop(0) + } + } + private fun testDns(): Dns = Dns { hostname -> - if (hostname == "garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) + if (hostname == "typetype-garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) } } From 9dbe1998f95cec7f5a06eaf0f12b3123cbe377e5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 12 Aug 2026 17:53:03 +0200 Subject: [PATCH 24/65] fix: preserve scheduled live feed ordering --- .../services/SubscriptionFeedOrderer.kt | 17 ++++++++++--- .../server/services/VideoItemSchedule.kt | 2 +- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 2 +- .../server/SubscriptionFeedOrdererTest.kt | 25 +++++++++++++------ 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt index c9cb2f66..a1dc2b9d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -10,13 +10,13 @@ internal class SubscriptionFeedOrderer { ): SubscriptionFeedOrdering { val previousByKey = previous?.videos.orEmpty().associateBy(VideoItem::subscriptionFeedKey) val promotedAt = buildMap { - videos.filter(VideoItem::isLive).forEach { video -> + videos.filter { it.isLiveOrUpcomingAt(refreshedAt) }.forEach { video -> val key = video.subscriptionFeedKey() val previousVideo = previousByKey[key] val promotion = when { - previousVideo?.isLive == true -> previous?.livePromotedAt?.get(key) - ?: previous?.generatedAt - ?: refreshedAt + previous == null || previousVideo == null -> refreshedAt + samePromotionPhase(video, previousVideo, previous) -> + previous.livePromotedAt[key] ?: previous.generatedAt else -> refreshedAt } put(key, promotion) @@ -30,6 +30,15 @@ internal class SubscriptionFeedOrderer { return SubscriptionFeedOrdering(ordered, promotedAt) } + private fun samePromotionPhase( + video: VideoItem, + previousVideo: VideoItem, + previous: SubscriptionFeedSnapshot, + ): Boolean = when { + video.isLive -> previousVideo.isLive + else -> previousVideo.isUpcomingAt(previous.generatedAt) + } + private fun VideoItem.feedTimestamp(): Long = when { uploaded >= 0L -> uploaded publishedAt != null && publishedAt >= 0L -> publishedAt diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt index 977b3cd9..25782930 100644 --- a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt +++ b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -3,6 +3,6 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem internal fun VideoItem.isUpcomingAt(now: Long): Boolean = - !isPostLive && duration < 0 && RssVideoMetadata.publishedAtMillis(this) > now + !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index c92392f2..90a2f6f0 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -49,7 +49,7 @@ class SubscriptionFeedLiveVisibilityRoutesTest { coEvery { channelService.getChannel(any(), null) } returns channel( video(4_000L, url = "https://youtube.com/watch?v=live", live = true), video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( - duration = -1L, + duration = 0L, publishedAt = System.currentTimeMillis() + 86_400_000L, ), video(3_000L, url = "https://youtube.com/watch?v=normal-1"), diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt index 8dd8dd1c..77956590 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt @@ -10,20 +10,31 @@ class SubscriptionFeedOrdererTest { private val orderer = SubscriptionFeedOrderer() @Test - fun `scheduled livestream follows normal chronology`() { - val scheduled = video(2_000L, url = "scheduled") - val recent = video(3_000L, url = "recent") + fun `scheduled livestream is promoted once then newer uploads pass it`() { + val discoveredAt = 1_800_000_000_000L + val scheduled = video(discoveredAt + 86_400_000L, url = "scheduled").copy(duration = 0L) + val first = orderer.order( + listOf(scheduled, video(discoveredAt - 1_000L, url = "existing")), + previous = null, + refreshedAt = discoveredAt, + ) + assertEquals(listOf("scheduled", "existing"), first.videos.map { it.url }) + val previous = snapshot(discoveredAt, first.videos, first.livePromotedAt) - val result = orderer.order(listOf(scheduled, recent), previous = null, refreshedAt = 10_000L) + val result = orderer.order( + listOf(scheduled, video(discoveredAt + 1_000L, url = "recent")), + previous, + refreshedAt = discoveredAt + 2_000L, + ) assertEquals(listOf("recent", "scheduled"), result.videos.map { it.url }) - assertEquals(emptyMap(), result.livePromotedAt) + assertEquals(discoveredAt, result.livePromotedAt["scheduled"]) } @Test fun `scheduled to live transition is promoted once`() { - val scheduled = video(2_000L, url = "live") - val previous = snapshot(5_000L, listOf(scheduled)) + val scheduled = video(20_000L, url = "live").copy(duration = 0L) + val previous = snapshot(5_000L, listOf(scheduled), mapOf("live" to 5_000L)) val live = video(-1L, url = "live", live = true) val promoted = orderer.order(listOf(video(8_000L), live), previous, refreshedAt = 10_000L) From 1045b57bb8c76dac2a084b70c28794715b8b2a07 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 12 Aug 2026 18:06:10 +0200 Subject: [PATCH 25/65] feat: hide members-only content --- openapi/components/access-control.yaml | 1 + .../server/db/SettingsSchemaMigrations.kt | 1 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../server/routes/SubscriptionFeedRoutes.kt | 14 ++++++- .../services/SettingsPersistenceMappers.kt | 2 + .../server/services/SettingsService.kt | 15 ++++++-- .../services/SubscriptionFeedService.kt | 6 ++- .../services/SubscriptionFeedSnapshot.kt | 38 +++++++++++++++---- .../SettingsPrivacyControlsRoutesTest.kt | 4 +- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 34 +++++++++++++++++ 11 files changed, 102 insertions(+), 15 deletions(-) diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 900de1fe..c06ed269 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -34,6 +34,7 @@ SettingsItem: hideComments: { type: boolean, default: false } hideShorts: { type: boolean, default: false } hideSubscriptionLiveStreams: { type: boolean, default: false } + hideMembersOnlyContent: { type: boolean, default: false } accessMode: type: string enum: [unrestricted, allow_list] diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index 45566d0f..18618c96 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -25,6 +25,7 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_comments BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_shorts BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_subscription_live_streams BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_members_only_content BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 4744441f..c5abec12 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -33,6 +33,7 @@ object SettingsTable : Table("settings") { val hideComments = bool("hide_comments").default(false) val hideShorts = bool("hide_shorts").default(false) val hideSubscriptionLiveStreams = bool("hide_subscription_live_streams").default(false) + val hideMembersOnlyContent = bool("hide_members_only_content").default(false) val disableWatchHistory = bool("disable_watch_history").default(false) val deArrowEnabled = bool("dearrow_enabled").default(false) val deArrowTitleMode = text("dearrow_title_mode").default("dearrow") diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index b6f46a83..0239bbc1 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -33,6 +33,7 @@ data class SettingsItem( val hideComments: Boolean = false, val hideShorts: Boolean = false, val hideSubscriptionLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, val disableWatchHistory: Boolean = false, val deArrowEnabled: Boolean = false, val deArrowTitleMode: String = "dearrow", diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 07154402..e382da1c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.SubscriptionFeedPreparingResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionFeedVisibility import dev.typetype.server.services.SettingsService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -24,9 +25,18 @@ fun Route.subscriptionFeedRoutes( val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 val cursor = call.request.queryParameters["cursor"] - val hideLiveStreams = settingsService?.hidesSubscriptionLiveStreams(userId) ?: false + val visibility = settingsService?.subscriptionFeedVisibility(userId) ?: SubscriptionFeedVisibility() call.response.headers.append(HttpHeaders.CacheControl, "no-store") - when (val result = feedService.getPage(userId, page, limit, cursor, hideLiveStreams)) { + when ( + val result = feedService.getPage( + userId, + page, + limit, + cursor, + visibility.hideLiveStreams, + visibility.hideMembersOnlyContent, + ) + ) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) is SubscriptionFeedPageResult.Preparing -> { call.response.headers.append(HttpHeaders.RetryAfter, "1") diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index f40a0237..813900a2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -46,6 +46,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( hideComments = this[SettingsTable.hideComments], hideShorts = this[SettingsTable.hideShorts], hideSubscriptionLiveStreams = this[SettingsTable.hideSubscriptionLiveStreams], + hideMembersOnlyContent = this[SettingsTable.hideMembersOnlyContent], disableWatchHistory = this[SettingsTable.disableWatchHistory], deArrowEnabled = this[SettingsTable.deArrowEnabled], deArrowTitleMode = this[SettingsTable.deArrowTitleMode], @@ -84,6 +85,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.hideComments] = settings.hideComments this[SettingsTable.hideShorts] = settings.hideShorts this[SettingsTable.hideSubscriptionLiveStreams] = settings.hideSubscriptionLiveStreams + this[SettingsTable.hideMembersOnlyContent] = settings.hideMembersOnlyContent this[SettingsTable.disableWatchHistory] = settings.disableWatchHistory this[SettingsTable.deArrowEnabled] = settings.deArrowEnabled this[SettingsTable.deArrowTitleMode] = settings.deArrowTitleMode diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index 1dffe3c3..d609931b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,9 +35,13 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } - suspend fun hidesSubscriptionLiveStreams(userId: String): Boolean = DatabaseFactory.query { - SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull() - ?.get(SettingsTable.hideSubscriptionLiveStreams) ?: false + internal suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { + SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { + SubscriptionFeedVisibility( + hideLiveStreams = it[SettingsTable.hideSubscriptionLiveStreams], + hideMembersOnlyContent = it[SettingsTable.hideMembersOnlyContent], + ) + } ?: SubscriptionFeedVisibility() } suspend fun getAccessModePolicy(userId: String): AccessModePolicy = DatabaseFactory.query { @@ -50,4 +54,9 @@ class SettingsService { } } +internal data class SubscriptionFeedVisibility( + val hideLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, +) + data class AccessModePolicy(val accessMode: String, val adminManaged: Boolean) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index f641e2ec..de1da91c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -37,6 +37,7 @@ class SubscriptionFeedService( limit: Int, cursor: String?, hideLiveStreams: Boolean = false, + hideMembersOnlyContent: Boolean = false, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -53,6 +54,9 @@ class SubscriptionFeedService( if (cursorState != null && cursorState.hideLiveStreams != hideLiveStreams) { return SubscriptionFeedPageResult.InvalidCursor } + if (cursorState != null && cursorState.hideMembersOnlyContent != hideMembersOnlyContent) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current @@ -61,7 +65,7 @@ class SubscriptionFeedService( } val offset = cursorState?.offset ?: page * limit return SubscriptionFeedPageResult.Ready( - snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams), + snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams, hideMembersOnlyContent), ) } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 2f7fd2ec..c752034a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -21,13 +21,20 @@ private data class SubscriptionFeedCursor( val offset: Int, val limit: Int, val hideLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, ) internal object SubscriptionFeedCursorCodec { - fun encode(generation: Long, offset: Int, limit: Int, hideLiveStreams: Boolean): String { + fun encode( + generation: Long, + offset: Int, + limit: Int, + hideLiveStreams: Boolean, + hideMembersOnlyContent: Boolean, + ): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams), + SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams, hideMembersOnlyContent), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -36,7 +43,15 @@ internal object SubscriptionFeedCursorCodec { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } - ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit, it.hideLiveStreams) } + ?.let { + SubscriptionFeedCursorState( + it.generation, + it.offset, + it.limit, + it.hideLiveStreams, + it.hideMembersOnlyContent, + ) + } }.getOrNull() } @@ -45,6 +60,7 @@ internal data class SubscriptionFeedCursorState( val offset: Int, val limit: Int, val hideLiveStreams: Boolean, + val hideMembersOnlyContent: Boolean, ) internal fun SubscriptionFeedSnapshot.page( @@ -52,16 +68,22 @@ internal fun SubscriptionFeedSnapshot.page( limit: Int, refreshing: Boolean, hideLiveStreams: Boolean = false, + hideMembersOnlyContent: Boolean = false, ): SubscriptionFeedResponse { - val visibleVideos = if (hideLiveStreams) { - videos.filterNot { it.isLiveOrUpcomingAt(generatedAt) } - } else { - videos + val visibleVideos = videos.filterNot { video -> + (hideLiveStreams && video.isLiveOrUpcomingAt(generatedAt)) || + (hideMembersOnlyContent && video.requiresMembership) } val from = offset.coerceAtMost(visibleVideos.size) val to = minOf(from + limit, visibleVideos.size) val nextpage = if (to < visibleVideos.size) { - SubscriptionFeedCursorCodec.encode(generation, to, limit, hideLiveStreams) + SubscriptionFeedCursorCodec.encode( + generation, + to, + limit, + hideLiveStreams, + hideMembersOnlyContent, + ) } else { null } diff --git a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt index cf6891bd..c91360e4 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt @@ -64,6 +64,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideComments\":false", "\"hideShorts\":false", "\"hideSubscriptionLiveStreams\":false", + "\"hideMembersOnlyContent\":false", ), ) } @@ -89,6 +90,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideComments\":true", "\"hideShorts\":true", "\"hideSubscriptionLiveStreams\":true", + "\"hideMembersOnlyContent\":true", ), ) } @@ -108,6 +110,6 @@ class SettingsPrivacyControlsRoutesTest { values.forEach { assertTrue(body.contains(it)) } private fun settingsBody(sponsorBlockMode: String = "mark_only"): String = """ - {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true} + {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true,"hideMembersOnlyContent":true} """.trimIndent() } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index 90a2f6f0..c2c0ec10 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -107,6 +107,40 @@ class SubscriptionFeedLiveVisibilityRoutesTest { assertEquals(listOf("replay"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) } + @Test + fun `account setting hides members only videos before pagination`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=members").copy(requiresMembership = true), + video(3_000L, url = "https://youtube.com/watch?v=public-1"), + video(2_000L, url = "https://youtube.com/watch?v=public-2"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideMembersOnlyContent = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val first = readPage(requestFeed(limit = 1)) + val second = readPage(requestFeed(limit = 1, cursor = requireNotNull(first.nextpage))) + + assertEquals(listOf("public-1"), first.videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("public-2"), second.videos.map { it.url.substringAfter("v=") }) + assertTrue(second.nextpage == null) + } + + @Test + fun `cursor is rejected after members only visibility changes`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(readPage(requestFeed(limit = 1)).nextpage) + + settingsService.upsert(TEST_USER_ID, SettingsItem(hideMembersOnlyContent = true)) + + assertEquals(HttpStatusCode.BadRequest, requestFeed(limit = 1, cursor = cursor).status) + } + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json() } From 44941e516f320278db647c76081546b074ad21ee Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 09:06:51 +0200 Subject: [PATCH 26/65] fix: prevent stale OIDC bootstrap status --- src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt | 2 +- src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt index ac9f1f4a..c3b59832 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt @@ -23,7 +23,7 @@ fun Route.registerRoutes( get("/auth/register/status") { val bootstrapAvailable = !authService.hasAdmin() val settings = adminSettingsService.get() - call.respond( + call.respondNoStore( RegisterStatusResponse( allowRegistration = settings.allowRegistration, bootstrapAvailable = bootstrapAvailable, diff --git a/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt b/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt index 01390d34..6c4de0d2 100644 --- a/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt +++ b/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt @@ -11,6 +11,7 @@ import io.ktor.client.request.post import io.ktor.client.request.setBody import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.contentType import io.ktor.serialization.kotlinx.json.json @@ -79,6 +80,7 @@ class RegistrationSettingsTest { } val response = client.get("/auth/register/status") assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store, no-cache, must-revalidate, max-age=0", response.headers[HttpHeaders.CacheControl]) assertEquals("""{"allowRegistration":false,"bootstrapAvailable":true,"localLoginEnabled":true}""", response.bodyAsText()) } From 8261d9001c2d1cbbca2dc2bf2cabb10f599c0ee9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 09:06:51 +0200 Subject: [PATCH 27/65] feat: add notification popup preference --- openapi/components/access-control.yaml | 1 + .../kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt | 1 + .../kotlin/dev/typetype/server/db/tables/SettingsTable.kt | 1 + src/main/kotlin/dev/typetype/server/models/SettingsItem.kt | 1 + .../typetype/server/services/SettingsPersistenceMappers.kt | 2 ++ src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt | 4 +++- 6 files changed, 9 insertions(+), 1 deletion(-) diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index c06ed269..5e3d7d25 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -13,6 +13,7 @@ SettingsItem: autoplay: { type: boolean, default: true } volume: { type: number, format: double, default: 1.0 } muted: { type: boolean, default: false } + notificationPopupsEnabled: { type: boolean, default: true } subtitlesEnabled: { type: boolean, default: false } defaultSubtitleLanguage: { type: string, default: "" } defaultAudioLanguage: { type: string, default: "" } diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index 18618c96..e49baf63 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -5,6 +5,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager object SettingsSchemaMigrations { fun apply() { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subtitles_enabled BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS notification_popups_enabled BOOLEAN NOT NULL DEFAULT true") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_playback_speed DOUBLE PRECISION NOT NULL DEFAULT 1.0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_subtitle_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_audio_language TEXT NOT NULL DEFAULT ''") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index c5abec12..253b348f 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -12,6 +12,7 @@ object SettingsTable : Table("settings") { val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) val volume = double("volume").default(1.0) val muted = bool("muted").default(false) + val notificationPopupsEnabled = bool("notification_popups_enabled").default(true) val subtitlesEnabled = bool("subtitles_enabled").default(false) val defaultSubtitleLanguage = text("default_subtitle_language").default("") val defaultAudioLanguage = text("default_audio_language").default("") diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 0239bbc1..5d5d7447 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -12,6 +12,7 @@ data class SettingsItem( val skipPlaylistAutoplayScreen: Boolean = false, val volume: Double = 1.0, val muted: Boolean = false, + val notificationPopupsEnabled: Boolean = true, val subtitlesEnabled: Boolean = false, val defaultSubtitleLanguage: String = "", val defaultAudioLanguage: String = "", diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 813900a2..e2087004 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -25,6 +25,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], volume = this[SettingsTable.volume], muted = this[SettingsTable.muted], + notificationPopupsEnabled = this[SettingsTable.notificationPopupsEnabled], subtitlesEnabled = this[SettingsTable.subtitlesEnabled], defaultSubtitleLanguage = this[SettingsTable.defaultSubtitleLanguage], defaultAudioLanguage = this[SettingsTable.defaultAudioLanguage], @@ -64,6 +65,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen this[SettingsTable.volume] = settings.volume this[SettingsTable.muted] = settings.muted + this[SettingsTable.notificationPopupsEnabled] = settings.notificationPopupsEnabled this[SettingsTable.subtitlesEnabled] = settings.subtitlesEnabled this[SettingsTable.defaultSubtitleLanguage] = settings.defaultSubtitleLanguage this[SettingsTable.defaultAudioLanguage] = settings.defaultAudioLanguage diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index 883e3808..f2fc67f3 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -59,6 +59,7 @@ class SettingsRoutesTest { val body = response.bodyAsText() assertTrue(body.contains("\"volume\":1.0")) assertTrue(body.contains("\"muted\":false")) + assertTrue(body.contains("\"notificationPopupsEnabled\":true")) assertTrue(body.contains("\"defaultLandingPage\":\"home\"")) assertTrue(body.contains("\"defaultPlaybackSpeed\":1.0")) } @@ -81,11 +82,12 @@ class SettingsRoutesTest { client.put("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - setBody("""{"defaultService":0,"defaultQuality":"720p","defaultLandingPage":"subscriptions","autoplay":false,"volume":0.5,"muted":true}""") + setBody("""{"defaultService":0,"defaultQuality":"720p","defaultLandingPage":"subscriptions","autoplay":false,"volume":0.5,"muted":true,"notificationPopupsEnabled":false}""") } val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() assertTrue(body.contains("\"volume\":0.5")) assertTrue(body.contains("\"muted\":true")) + assertTrue(body.contains("\"notificationPopupsEnabled\":false")) assertTrue(body.contains("\"defaultQuality\":\"720p\"")) assertTrue(body.contains("\"defaultLandingPage\":\"subscriptions\"")) } From 2c5e0a4275cc97e6540d10088fb843eafb076556 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 13:15:19 +0200 Subject: [PATCH 28/65] feat: support account-bound SABR token pairs --- .../AuthenticatedYoutubeVisitorData.kt | 27 +++++++++++++++ .../server/services/SabrTokenBundle.kt | 31 ++++++++++++++++- .../services/TypetypeTokenSabrTokenClient.kt | 34 +++++++++++++++++++ ...etypeTokenYoutubeSessionPoTokenProvider.kt | 22 +----------- .../TypetypeYoutubeSessionPoTokenProvider.kt | 7 ++-- .../TypetypeTokenSabrTokenClientTest.kt | 28 +++++++++++++-- 6 files changed, 123 insertions(+), 26 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt new file mode 100644 index 00000000..2c2d55db --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper + +internal object AuthenticatedYoutubeVisitorData { + fun fetch( + localization: Localization = Localization("en", "US"), + contentCountry: ContentCountry = ContentCountry("US"), + ): String { + val headers = HashMap>() + YoutubeParsingHelper.addYoutubeHeaders(headers) + headers["Content-Type"] = listOf("application/json") + YoutubeParsingHelper.addLoggedInHeaders(headers) + return YoutubeParsingHelper.getVisitorDataFromInnertube( + InnertubeClientRequestInfo.ofWebClient(), + localization, + contentCountry, + headers, + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt index e2be83a9..001f71a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import org.json.JSONObject +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import java.util.Base64 @@ -11,6 +12,8 @@ internal class SabrTokenBundle( val visitorData: String, val videoBoundPoToken: String, val videoBoundPoTokenBytes: ByteArray, + val sessionBinding: String? = null, + val sessionBoundPoToken: String? = null, ) { val visitorPoToken: String = visitorBoundPoToken val visitorPoTokenBytes: ByteArray = visitorBoundPoTokenBytes @@ -33,6 +36,26 @@ internal class SabrTokenBundle( ) }.getOrNull() + fun fromSessionResponse( + videoId: String, + sessionBinding: String, + json: JSONObject, + ): SabrTokenBundle? { + val base = fromResponse(videoId, json) ?: return null + val sessionBoundPoToken = json.optString("sessionBoundPoToken").takeIf(String::isNotBlank) + ?: return null + return SabrTokenBundle( + videoId = base.videoId, + visitorBoundPoToken = base.visitorBoundPoToken, + visitorBoundPoTokenBytes = base.visitorBoundPoTokenBytes, + visitorData = base.visitorData, + videoBoundPoToken = base.videoBoundPoToken, + videoBoundPoTokenBytes = base.videoBoundPoTokenBytes, + sessionBinding = sessionBinding, + sessionBoundPoToken = sessionBoundPoToken, + ) + } + private fun decodeBase64Url(value: String): ByteArray { val padded = value + "=".repeat((4 - value.length % 4) % 4) return Base64.getUrlDecoder().decode(padded) @@ -40,8 +63,14 @@ internal class SabrTokenBundle( } } +internal fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = + YoutubeSessionPoToken(sessionBinding ?: visitorData, sessionBoundPoToken ?: visitorBoundPoToken) + internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = - takeIf { it.videoId == info.videoId && it.visitorData == info.visitorData } + takeIf { + it.videoId == info.videoId && + (it.visitorData == info.visitorData || it.sessionBinding == info.visitorData) + } ?.streamingPoTokenBytes ?.takeIf { it.isNotEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index a9447a42..6daab550 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -1,7 +1,9 @@ package dev.typetype.server.services import okhttp3.OkHttpClient +import okhttp3.MediaType.Companion.toMediaType import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -18,6 +20,34 @@ internal class TypetypeTokenSabrTokenClient( fun fetchBoundToken(binding: String): String? = fetch(binding, forceRefresh = false, refreshVideo = false, logIdentifier = false)?.videoBoundPoToken + fun fetchSession( + videoId: String, + sessionBinding: String, + refreshVideo: Boolean = false, + ): SabrTokenBundle? { + val body = JSONObject() + .put("videoId", videoId) + .put("sessionBinding", sessionBinding) + .put("refreshVideo", refreshVideo) + .toString() + .toRequestBody(JSON_MEDIA_TYPE) + val request = Request.Builder() + .url("${tokenServiceUrl.trimEnd('/')}/potoken/session") + .post(body) + .build() + return try { + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) return null + SabrTokenBundle.fromSessionResponse(videoId, sessionBinding, JSONObject(response.body.string())) + } + } catch (error: Exception) { + System.err.println( + "[TypetypeTokenSabrTokenClient] authenticated token fetch failed: ${error.message}", + ) + null + } + } + private fun fetch( binding: String, forceRefresh: Boolean, @@ -50,4 +80,8 @@ internal class TypetypeTokenSabrTokenClient( (if (forceRefresh) "&refresh=true" else "") + (if (refreshVideo) "&refreshVideo=true" else "") } + + private companion object { + val JSON_MEDIA_TYPE = "application/json".toMediaType() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt index abb8ca67..15b63405 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -3,8 +3,6 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider import java.security.MessageDigest @@ -20,7 +18,7 @@ internal class TypetypeTokenYoutubeSessionPoTokenProvider( TypetypeTokenSabrTokenClient(tokenServiceUrl).let { client -> { binding -> client.fetchBoundToken(binding) } }, - ::fetchAuthenticatedVisitorData, + AuthenticatedYoutubeVisitorData::fetch, ) @Volatile private var cached: CachedToken? = null @@ -76,23 +74,5 @@ internal class TypetypeTokenYoutubeSessionPoTokenProvider( private companion object { const val TOKEN_TTL_MS = 6L * 60L * 60L * 1000L - fun fetchAuthenticatedVisitorData( - localization: Localization, - contentCountry: ContentCountry, - ): String { - val headers = HashMap>() - YoutubeParsingHelper.addYoutubeHeaders(headers) - headers["Content-Type"] = listOf("application/json") - YoutubeParsingHelper.addLoggedInHeaders(headers) - return YoutubeParsingHelper.getVisitorDataFromInnertube( - InnertubeClientRequestInfo.ofWebClient(), - localization, - contentCountry, - headers, - YoutubeParsingHelper.YOUTUBEI_V1_URL, - null, - false, - ) - } } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index 72bff27b..7ef95956 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -9,9 +9,12 @@ internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenPro private val scopedToken = ThreadLocal() fun withToken(token: SabrTokenBundle, block: () -> T): T { + return withToken(token.youtubeSessionPoToken(), block) + } + + fun withToken(token: YoutubeSessionPoToken, block: () -> T): T { val previous = scopedToken.get() - val sessionToken = YoutubeSessionPoToken(token.visitorData, token.visitorBoundPoToken) - scopedToken.set(sessionToken) + scopedToken.set(token) return try { block() } finally { diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt index 54b3a704..6022321e 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt @@ -5,8 +5,10 @@ import io.mockk.mockk import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.Protocol +import okhttp3.Request import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONObject import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -79,16 +81,36 @@ class TypetypeTokenSabrTokenClientTest { assertNull(url.queryParameter("refreshVideo")) } + @Test + fun sessionFetchPostsOneExplicitlyBoundTokenPair(): Unit { + val recorder = PotokenRequestRecorder(SESSION_TOKEN_JSON) + val client = TypetypeTokenSabrTokenClient("https://token.example", recorder.client) + + val token = client.fetchSession("video", "connected-visitor", refreshVideo = true) + + assertNotNull(token) + val request = recorder.requests.single() + assertEquals("POST", request.method) + assertEquals("/potoken/session", request.url.encodedPath) + val body = JSONObject(request.body!!.let { body -> okio.Buffer().also(body::writeTo).readUtf8() }) + assertEquals("video", body.getString("videoId")) + assertEquals("connected-visitor", body.getString("sessionBinding")) + assertEquals(true, body.getBoolean("refreshVideo")) + assertArrayEquals(byteArrayOf(2), token!!.streamingPoTokenBytesFor(info("connected-visitor"))) + assertNull(token.streamingPoTokenBytesFor(info("different-visitor"))) + } + private fun info(expectedVisitorData: String): YoutubeSabrInfo = mockk { every { videoId } returns "video" every { visitorData } returns expectedVisitorData } private class PotokenRequestRecorder(tokenJson: String = TOKEN_JSON) { - val urls = mutableListOf() + val requests = mutableListOf() + val urls: List get() = requests.map { it.url } val client: OkHttpClient = OkHttpClient.Builder() .addInterceptor(Interceptor { chain -> - urls += chain.request().url + requests += chain.request() Response.Builder() .request(chain.request()) .protocol(Protocol.HTTP_1_1) @@ -105,5 +127,7 @@ class TypetypeTokenSabrTokenClientTest { """{"visitorBoundPoToken":"AQ","visitorData":"visitor","videoBoundPoToken":"Ag"}""" const val MISMATCHED_TOKEN_JSON = """{"visitorBoundPoToken":"AQ","visitorData":"other-visitor","videoBoundPoToken":"Ag"}""" + const val SESSION_TOKEN_JSON = + """{"visitorBoundPoToken":"AQ","visitorData":"public","videoBoundPoToken":"Ag","sessionBoundPoToken":"Aw"}""" } } From e73c00eb9c8e4b3e7d26c8dcd293fc6246494c83 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 13:15:28 +0200 Subject: [PATCH 29/65] fix: use connected accounts for SABR playback --- .../dev/typetype/server/ApplicationRoutes.kt | 1 + .../server/ExtractionServiceRegistry.kt | 6 ++ .../dev/typetype/server/ServiceRegistry.kt | 1 + .../server/routes/SabrPlaybackHandler.kt | 8 +- .../dev/typetype/server/routes/SabrRoutes.kt | 3 + .../services/AuthenticatedSabrInfoService.kt | 82 +++++++++++++++++++ .../server/services/SabrDownloadStreamer.kt | 4 +- .../services/SabrPlaybackInfoResolver.kt | 30 +++++++ .../services/SabrPlaybackSessionService.kt | 1 + .../server/services/SabrPreparedInfo.kt | 1 + .../server/services/SabrPreparedSource.kt | 6 ++ .../server/services/SabrSessionFactory.kt | 2 + .../server/services/SabrSessionHolder.kt | 1 + .../server/services/SabrSessionPump.kt | 2 +- .../server/services/SabrSessionStore.kt | 8 +- .../SabrUnauthorizedResponseRecovery.kt | 4 +- 16 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 41e04025..638e3971 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -92,6 +92,7 @@ internal fun Application.installApplicationRoutes( svc.accessControlService, adminSettingsService, svc.audioOnlyMediaTokenService, + svc.authenticatedSabrInfoService, ) } downloaderGatewayRoutes(downloaderGatewayService) diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 1fe29a3e..123ab4e0 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.cache.DragonflyService import dev.typetype.server.services.BilibiliRelatedService import dev.typetype.server.services.BilibiliTrendingService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.CachedChannelService import dev.typetype.server.services.CachedCommentService import dev.typetype.server.services.CachedManifestService @@ -33,6 +34,7 @@ import dev.typetype.server.services.SabrBootstrapStreamService import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.SignedHlsManifestTokenService import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient +import dev.typetype.server.services.TypetypeTokenSabrTokenClient import dev.typetype.server.services.YouTubeSubtitleService import dev.typetype.server.services.YouTubeSubtitleCache import dev.typetype.server.services.YouTubeSubtitleDeliveryService @@ -104,6 +106,10 @@ internal class ExtractionServiceRegistry( YoutubePlayerClient.MWEB, ) val youtubeSessionService = YoutubeSessionService(youtubeSessionSecret?.let(YoutubeSessionCrypto::fromSecret)) + val authenticatedSabrInfoService = AuthenticatedSabrInfoService( + youtubeSessionService, + TypetypeTokenSabrTokenClient(subtitleServiceUrl, httpClient), + ) private val hlsTokenService = youtubeSessionSecret?.let(::SignedHlsManifestTokenService) private val tokenYoutubeSessionClient = TypetypeTokenYoutubeSessionClient(subtitleServiceUrl, httpClient) val youtubeSessionStreamService = hlsTokenService?.let { diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index a6ee84ed..42341834 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -57,6 +57,7 @@ internal class ServiceRegistry( youtubeProxySelector, ) val youtubeSessionService = extraction.youtubeSessionService + val authenticatedSabrInfoService = extraction.authenticatedSabrInfoService val youtubeSessionStreamService = extraction.youtubeSessionStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index c4abe087..ab587f32 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -4,10 +4,12 @@ import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.AuthService import dev.typetype.server.services.SabrPreparedInfo import dev.typetype.server.services.SabrPlaybackSegmentResult import dev.typetype.server.services.SabrPlaybackSessionService +import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService @@ -23,15 +25,17 @@ internal class SabrPlaybackHandler( private val authService: AuthService?, private val accessControlService: AccessControlService?, private val adminSettingsService: AdminSettingsService?, + authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, ) { private val playbackService = SabrPlaybackSessionService(sabrSessionStore) + private val infoResolver = SabrPlaybackInfoResolver(sabrSessionStore, authenticatedSabrInfoService) suspend fun create(call: ApplicationCall, videoId: String) { val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return if (!validateAccess(call, videoId, access)) return val request = call.playbackRequest() val startTimeMs = request.effectiveStartTimeMs() - val prepared = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) + val prepared = infoResolver.initial(access.userId, videoId, startTimeMs) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = selectAudio(call, prepared, request) ?: return val video = selectVideo(call, prepared, request) ?: return @@ -58,7 +62,7 @@ internal class SabrPlaybackHandler( val preparation = playbackService.seekExisting(holder, playerTimeMs, request.audioOnly) return respondPrepared(call, holder, holder.key.videoId, preparation.startTimeMs, preparation.ready) } - val prepared = sabrSessionStore.fetchInfo(holder.key.videoId, playerTimeMs, cachedFirst = true) + val prepared = infoResolver.replacement(holder, playerTimeMs) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = SabrFormatSelector.audio( prepared.info, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt index 2f4abe7f..274f5298 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import io.ktor.http.HttpStatusCode @@ -20,6 +21,7 @@ internal fun Route.sabrRoutes( accessControlService: AccessControlService?, adminSettingsService: AdminSettingsService?, audioOnlyTokenService: AudioOnlyMediaTokenService?, + authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, ) { val sessionHandler = SabrSessionDescriptorHandler( sabrSessionStore, @@ -49,6 +51,7 @@ internal fun Route.sabrRoutes( authService, accessControlService, adminSettingsService, + authenticatedSabrInfoService, ) val playbackStateHandler = SabrPlaybackStateHandler(sabrSessionStore) val playbackWindowHandler = SabrPlaybackWindowHandler(sabrSessionStore) diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt new file mode 100644 index 00000000..9e0dee20 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe +import org.slf4j.LoggerFactory + +internal class AuthenticatedSabrInfoService( + private val youtubeSessionService: YoutubeSessionService, + private val tokenClient: TypetypeTokenSabrTokenClient, + private val visitorDataFetcher: () -> String = AuthenticatedYoutubeVisitorData::fetch, + private val probe: AuthenticatedSabrProbe = PipePipeAuthenticatedSabrProbe, +) { + suspend fun fetch(userId: String?, videoId: String): AuthenticatedSabrInfoResult { + if (userId == null || userId.startsWith("guest:")) return AuthenticatedSabrInfoResult.NotConnected + val credentials = youtubeSessionService.connectedCredentials(userId) + ?: return AuthenticatedSabrInfoResult.NotConnected + return try { + val prepared = YoutubeSessionTokenScope.withCredentials(credentials) { + withContext(Dispatchers.IO) { + val sessionBinding = visitorDataFetcher() + val token = tokenClient.fetchSession(videoId, sessionBinding) + ?: error("Token service did not return authenticated SABR tokens") + val info = probe.fetch(videoId, token.youtubeSessionPoToken()) + SabrPreparedInfo( + info = info, + initialToken = token, + source = SabrPreparedSource.AUTHENTICATED_YOUTUBE, + ).takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) + ?: error("Authenticated SABR response has no audio and video formats") + } + } + youtubeSessionService.markUsed(userId) + AuthenticatedSabrInfoResult.Ready(prepared) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + logger.warn( + "authenticated_sabr_probe event=failed videoId={} errorType={} error={}", + videoId, + error.javaClass.simpleName, + error.message, + ) + AuthenticatedSabrInfoResult.Failed + } + } + + private companion object { + val logger = LoggerFactory.getLogger(AuthenticatedSabrInfoService::class.java) + } +} + +internal sealed interface AuthenticatedSabrInfoResult { + data object NotConnected : AuthenticatedSabrInfoResult + data object Failed : AuthenticatedSabrInfoResult + data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult +} + +internal fun interface AuthenticatedSabrProbe { + fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo +} + +private object PipePipeAuthenticatedSabrProbe : AuthenticatedSabrProbe { + private val localization = Localization("en", "US") + private val contentCountry = ContentCountry("US") + + override fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo = + YoutubeSabrProbe.fetchSabrInfo( + videoId, + YoutubeSabrClientProfile.WEB, + localization, + contentCountry, + token.poToken, + token.visitorData, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt index 6024be9d..15c3ed98 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -17,7 +17,9 @@ internal class SabrDownloadStreamer( private val pumpTimeoutMs: Long = PUMP_TIMEOUT_MS, ) { private val localization = Localization("en", "US") - private val unauthorizedRecovery = SabrUnauthorizedResponseRecovery(store::refreshVideoPoToken) + private val unauthorizedRecovery = SabrUnauthorizedResponseRecovery { holder -> + store.refreshVideoPoToken(holder.key.videoId) + } suspend fun stream( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt new file mode 100644 index 00000000..4486e836 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -0,0 +1,30 @@ +package dev.typetype.server.services + +internal class SabrPlaybackInfoResolver( + private val sessionStore: SabrSessionStore, + private val authenticatedInfoService: AuthenticatedSabrInfoService?, +) { + suspend fun initial( + userId: String?, + videoId: String, + startTimeMs: Long, + ): SabrPreparedInfo? = when (val authenticated = authenticatedInfoService?.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared + AuthenticatedSabrInfoResult.Failed -> null + AuthenticatedSabrInfoResult.NotConnected, null -> + sessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) + } + + suspend fun replacement(holder: SabrSessionHolder, startTimeMs: Long): SabrPreparedInfo? { + if (holder.source == SabrPreparedSource.PUBLIC) { + return sessionStore.fetchInfo(holder.key.videoId, startTimeMs, cachedFirst = true) + } + return when (val authenticated = authenticatedInfoService?.fetch(holder.key.userId, holder.key.videoId)) { + is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared + AuthenticatedSabrInfoResult.Failed, + AuthenticatedSabrInfoResult.NotConnected, + null, + -> null + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index 2bd6732d..dadf06b0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -29,6 +29,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS purpose = SabrSessionPurpose.PLAYBACK, audioOnly = audioOnly, initialGeneration = initialGeneration, + source = prepared.source, ) if (isLive || prepared.isLive) holder.markExpectedLive() if (holder.expectsLive()) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt index 98de8cd2..23d59c87 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt @@ -7,6 +7,7 @@ internal class SabrPreparedInfo( val initialToken: SabrTokenBundle?, val isLive: Boolean = false, val isLiveContent: Boolean = false, + val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) internal fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt b/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt new file mode 100644 index 00000000..93b526e1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.services + +internal enum class SabrPreparedSource { + PUBLIC, + AUTHENTICATED_YOUTUBE, +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt index 3269b6dc..5e6ac047 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt @@ -16,6 +16,7 @@ internal class SabrSessionFactory( sessionToken: String, initialToken: SabrTokenBundle?, initialGeneration: Long, + source: SabrPreparedSource, ): SabrSessionHolder { val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, initialToken) val sessionInfo = if (key.sourceId == null) info else SabrSessionIdentity.fresh(info) @@ -34,6 +35,7 @@ internal class SabrSessionFactory( Instant.now(), initialToken, initialGeneration = initialGeneration, + source = source, ).also { it.setPlayerTimeMs(key.startTimeMs) } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index 2d0d7ecb..626e6b2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -23,6 +23,7 @@ internal class SabrSessionHolder( @Volatile var playerContextToken: SabrTokenBundle? = null, val pumpMutex: Mutex = Mutex(), initialGeneration: Long = 0L, + val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) { private val readerPositions = ConcurrentHashMap() private val lastServedSequences = ConcurrentHashMap() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 75f8946d..5b0e12ac 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -9,7 +9,7 @@ import java.time.Instant internal class SabrSessionPump( private val segmentCache: SabrSegmentCache? = null, - refreshPoToken: (String) -> SabrTokenBundle? = { null }, + refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle? = { null }, ) { private val loop = SabrSessionPumpLoop( unauthorizedRecovery = SabrUnauthorizedResponseRecovery(refreshPoToken), diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index a76ea88a..5f666549 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -27,8 +27,10 @@ internal class SabrSessionStore( ) { private val registry = SabrSessionRegistry() private val segmentCache = SabrSegmentCache() - private val pump = SabrSessionPump(segmentCache) { videoId -> - tokenClient.fetch(videoId, refreshVideo = true) + private val pump = SabrSessionPump(segmentCache) { holder -> + val binding = holder.playerContextToken?.sessionBinding + if (binding == null) tokenClient.fetch(holder.key.videoId, refreshVideo = true) + else tokenClient.fetchSession(holder.key.videoId, binding, refreshVideo = true) } private val warmer = SabrPlaybackWarmer() private val infoFetcher = SabrInfoFetcher(tokenClient, sessionClient, sharedCache = initCache) @@ -51,6 +53,7 @@ internal class SabrSessionStore( purpose: SabrSessionPurpose = SabrSessionPurpose.MANIFEST, audioOnly: Boolean = false, initialGeneration: Long = 0L, + source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ): SabrSessionHolder { val sessionToken = SabrSessionTokenGenerator.newToken() val isolatedSourceId = sessionToken.takeIf { @@ -77,6 +80,7 @@ internal class SabrSessionStore( sessionToken, initialToken, initialGeneration, + source, ) val active = registry.put(key, holder) if (active !== holder) return active diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index 393988a8..8f3d0d12 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -3,12 +3,12 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException internal class SabrUnauthorizedResponseRecovery( - private val refreshPoToken: (String) -> SabrTokenBundle?, + private val refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle?, ) { fun verify(holder: SabrSessionHolder): Unit { val status = latestUnauthorizedStatus(holder.session.diagnosticTrace) ?: return if (!holder.markUnauthorizedRefreshAttempted()) throw unauthorized(status) - val refreshed = refreshPoToken(holder.key.videoId) ?: throw unauthorized(status) + val refreshed = refreshPoToken(holder) ?: throw unauthorized(status) val token = refreshed.streamingPoTokenBytesFor(holder.info) ?.takeUnless { holder.session.streamState.poToken?.contentEquals(it) == true } ?: throw unauthorized(status) From 0d1d89bd9b71a78f9d89f0ed2d4fe8baeb2e4e41 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 13:15:33 +0200 Subject: [PATCH 30/65] test: cover authenticated SABR info --- .../AuthenticatedSabrInfoServiceTest.kt | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt new file mode 100644 index 00000000..84d72c34 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -0,0 +1,138 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +class AuthenticatedSabrInfoServiceTest { + @Test + fun `connected account uses one authenticated token pair`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + val token = sessionToken() + val info = playableInfo() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns token + every { probe.fetch(VIDEO_ID, any()) } answers { + val supplied = secondArg() + assertEquals(SESSION_BINDING, supplied.visitorData) + assertEquals(SESSION_PO_TOKEN, supplied.poToken) + info + } + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + val result = service.fetch(USER_ID, VIDEO_ID) as AuthenticatedSabrInfoResult.Ready + + assertSame(info, result.prepared.info) + assertSame(token, result.prepared.initialToken) + assertEquals(SabrPreparedSource.AUTHENTICATED_YOUTUBE, result.prepared.source) + coVerify(exactly = 1) { sessions.markUsed(USER_ID) } + } + + @Test + fun `guest playback does not inspect connected credentials`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val service = AuthenticatedSabrInfoService(sessions, tokenClient) + + val result = service.fetch("guest:anonymous", VIDEO_ID) + + assertEquals(AuthenticatedSabrInfoResult.NotConnected, result) + coVerify(exactly = 0) { sessions.connectedCredentials(any()) } + verify(exactly = 0) { tokenClient.fetchSession(any(), any(), any()) } + } + + @Test + fun `authenticated probe failure is typed and does not mark session used`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns null + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + ) + + val result = service.fetch(USER_ID, VIDEO_ID) + + assertEquals(AuthenticatedSabrInfoResult.Failed, result) + coVerify(exactly = 0) { sessions.markUsed(any()) } + } + + @Test + fun `cancellation remains observable`() { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns sessionToken() + every { probe.fetch(VIDEO_ID, any()) } throws CancellationException("cancelled") + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + assertThrows(CancellationException::class.java) { + runTest { service.fetch(USER_ID, VIDEO_ID) } + } + } + + private fun playableInfo(): YoutubeSabrInfo = mockk { + every { formats } returns listOf( + mockk { + every { isAudio } returns true + every { isVideo } returns false + }, + mockk { + every { isAudio } returns false + every { isVideo } returns true + }, + ) + } + + private fun credentials(userId: String) = YoutubeSessionCredentials( + userId = userId, + fingerprint = "fingerprint-$userId", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + + private fun sessionToken() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "public-session-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "public-visitor", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + sessionBinding = SESSION_BINDING, + sessionBoundPoToken = SESSION_PO_TOKEN, + ) + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + const val SESSION_BINDING = "connected-visitor" + const val SESSION_PO_TOKEN = "connected-session-token" + } +} From 4d985f0a486d2ffa3ddd38515872da6fce00fadb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 13:15:33 +0200 Subject: [PATCH 31/65] test: preserve authenticated SABR sources --- .../services/SabrPlaybackInfoResolverTest.kt | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt new file mode 100644 index 00000000..251c237a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt @@ -0,0 +1,87 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class SabrPlaybackInfoResolverTest { + @Test + fun `connected account uses authenticated playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Ready(prepared) + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 0L) + + assertSame(prepared, result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `authenticated failure never falls back to public playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Failed + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 0L) + + assertNull(result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `account without YouTube connection uses public playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.NotConnected + coEvery { store.fetchInfo(VIDEO_ID, 5_000L, true, false) } returns prepared + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 5_000L) + + assertSame(prepared, result) + } + + @Test + fun `authenticated replacement remains authenticated`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + val holder = holder(SabrPreparedSource.AUTHENTICATED_YOUTUBE) + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Ready(prepared) + + val result = SabrPlaybackInfoResolver(store, authenticated).replacement(holder, 60_000L) + + assertSame(prepared, result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `authenticated replacement failure never changes source`() = runTest { + val store = mockk() + val authenticated = mockk() + val holder = holder(SabrPreparedSource.AUTHENTICATED_YOUTUBE) + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Failed + + val result = SabrPlaybackInfoResolver(store, authenticated).replacement(holder, 60_000L) + + assertNull(result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + private fun holder(source: SabrPreparedSource): SabrSessionHolder = mockk { + every { this@mockk.source } returns source + every { key } returns SabrSessionKey(VIDEO_ID, USER_ID, 140, null, 137, 0L) + } + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + } +} From c781e2175fabffca3d72e042da7773b4389ccd7e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 14:03:26 +0200 Subject: [PATCH 32/65] fix: route gated playback through YouTube sessions --- .../dev/typetype/server/ApplicationRoutes.kt | 3 ++ .../server/ApplicationStreamRoutes.kt | 3 ++ .../server/ExtractionServiceRegistry.kt | 4 ++ .../dev/typetype/server/ServiceRegistry.kt | 1 + .../routes/SabrPlaybackAccessValidator.kt | 25 +++++++++++ .../server/routes/SabrPlaybackHandler.kt | 14 +++--- .../dev/typetype/server/routes/SabrRoutes.kt | 4 ++ .../server/routes/StreamRouteDependencies.kt | 2 + .../typetype/server/routes/StreamRoutes.kt | 44 +++++++++++++++++-- .../services/StreamExtractionErrorMapper.kt | 5 ++- .../YoutubeSessionSabrStreamService.kt | 32 ++++++++++++++ .../services/YoutubeSessionStreamService.kt | 7 ++- 12 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 638e3971..1976d9cf 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -93,6 +93,9 @@ internal fun Application.installApplicationRoutes( adminSettingsService, svc.audioOnlyMediaTokenService, svc.authenticatedSabrInfoService, + svc.youtubeSessionSabrStreamService?.let { service -> + { userId, url -> service.getStreamInfo(userId, url) } + }, ) } downloaderGatewayRoutes(downloaderGatewayService) diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 681158ef..1c3fd0e5 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -31,6 +31,9 @@ internal fun Route.installStreamRoutes( blockedService = svc.blockedService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, + youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> + { userId, url -> service.getStreamInfo(userId, url) } + }, ) audioOnlyContractRoutes( streamService = svc.streamService, diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 123ab4e0..553531cc 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -55,6 +55,7 @@ import dev.typetype.server.services.YoutubeSessionCrypto import dev.typetype.server.services.YoutubeSessionHlsManifestService import dev.typetype.server.services.YoutubeSessionService import dev.typetype.server.services.YoutubeSessionStreamService +import dev.typetype.server.services.YoutubeSessionSabrStreamService import okhttp3.ConnectionPool import okhttp3.Dispatcher import okhttp3.OkHttpClient @@ -115,6 +116,9 @@ internal class ExtractionServiceRegistry( val youtubeSessionStreamService = hlsTokenService?.let { YoutubeSessionStreamService(authenticatedStreamService, youtubeSessionService, cache, it) } + val youtubeSessionSabrStreamService = youtubeSessionStreamService?.let { + YoutubeSessionSabrStreamService(it, authenticatedSabrInfoService) + } val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 42341834..3164b3d1 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -59,6 +59,7 @@ internal class ServiceRegistry( val youtubeSessionService = extraction.youtubeSessionService val authenticatedSabrInfoService = extraction.authenticatedSabrInfoService val youtubeSessionStreamService = extraction.youtubeSessionStreamService + val youtubeSessionSabrStreamService = extraction.youtubeSessionSabrStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt new file mode 100644 index 00000000..4cbf4b00 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.StreamService +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR +import dev.typetype.server.services.requiresYoutubeSession + +internal class SabrPlaybackAccessValidator( + private val publicStreamService: StreamService, + private val youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, +) { + suspend fun resolve(userId: String?, videoId: String): ExtractionResult { + val url = "https://www.youtube.com/watch?v=$videoId" + val publicResult = publicStreamService.getStreamInfo(url) + val authenticatedResult = userId?.let { id -> youtubeSessionStreamInfo?.invoke(id, url) } + if (authenticatedResult != null) return authenticatedResult + return if (publicResult.requiresYoutubeSession() && youtubeSessionStreamInfo != null) { + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE) + } else { + publicResult + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index ab587f32..0e544d4b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -2,6 +2,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthenticatedSabrInfoService @@ -26,9 +27,11 @@ internal class SabrPlaybackHandler( private val accessControlService: AccessControlService?, private val adminSettingsService: AdminSettingsService?, authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, + youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, ) { private val playbackService = SabrPlaybackSessionService(sabrSessionStore) private val infoResolver = SabrPlaybackInfoResolver(sabrSessionStore, authenticatedSabrInfoService) + private val accessValidator = SabrPlaybackAccessValidator(streamService, youtubeSessionStreamInfo) suspend fun create(call: ApplicationCall, videoId: String) { val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return @@ -125,20 +128,21 @@ internal class SabrPlaybackHandler( } private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { - if (!access.profile.enabled) return true - return when (val result = streamService.getStreamInfo("https://www.youtube.com/watch?v=$videoId")) { + return when (val result = accessValidator.resolve(access.userId, videoId)) { is ExtractionResult.Success -> { - if (access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) true else { + val allowed = !access.profile.enabled || + access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName) + if (allowed) true else { call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) false } } is ExtractionResult.Failure -> { - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) false } is ExtractionResult.BadRequest -> { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) false } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt index 274f5298..8f99d264 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt @@ -1,6 +1,8 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService @@ -22,6 +24,7 @@ internal fun Route.sabrRoutes( adminSettingsService: AdminSettingsService?, audioOnlyTokenService: AudioOnlyMediaTokenService?, authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, + youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, ) { val sessionHandler = SabrSessionDescriptorHandler( sabrSessionStore, @@ -52,6 +55,7 @@ internal fun Route.sabrRoutes( accessControlService, adminSettingsService, authenticatedSabrInfoService, + youtubeSessionStreamInfo, ) val playbackStateHandler = SabrPlaybackStateHandler(sabrSessionStore) val playbackWindowHandler = SabrPlaybackWindowHandler(sabrSessionStore) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 6598572a..177533c7 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -1,5 +1,6 @@ package dev.typetype.server.routes +import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService @@ -14,4 +15,5 @@ internal data class StreamRouteDependencies( val blockedService: BlockedService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, + val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 81f60626..9f315c08 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -10,8 +10,11 @@ import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.StreamService -import dev.typetype.server.services.filterBlocked +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked +import dev.typetype.server.services.requiresYoutubeSession import dev.typetype.server.services.withSabrManifestUrls import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -32,6 +35,7 @@ fun Route.streamRoutes( nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, + youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, ) { val dependencies = StreamRouteDependencies( @@ -41,6 +45,7 @@ fun Route.streamRoutes( blockedService = blockedService, publicHlsManifestTokenService = publicHlsManifestTokenService, sabrStreamContractFilter = sabrStreamContractFilter, + youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, ) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) streamRoute( @@ -83,7 +88,9 @@ private fun Route.streamRoute( ErrorResponse("Video is blocked", "content_blocked"), ) } - when (val result = streamService.getStreamInfo(url)) { + val publicResult = streamService.getStreamInfo(url) + val resolution = resolveStreamInfo(url, deliveryMode, access.userId, publicResult, dependencies) + when (val result = resolution.result) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) @@ -106,7 +113,11 @@ private fun Route.streamRoute( deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, ) - val data = if (!deliveryMode.isSabr() || dependencies.sabrStreamContractFilter == null) { + val data = if ( + !deliveryMode.isSabr() || + resolution.authenticated || + dependencies.sabrStreamContractFilter == null + ) { filtered } else { dependencies.sabrStreamContractFilter.invoke(url, filtered) @@ -131,6 +142,33 @@ private fun Route.streamRoute( } } +private data class StreamResolution( + val result: ExtractionResult, + val authenticated: Boolean = false, +) + +private suspend fun resolveStreamInfo( + url: String, + deliveryMode: StreamDeliveryMode, + userId: String?, + publicResult: ExtractionResult, + dependencies: StreamRouteDependencies, +): StreamResolution { + val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo + if (!deliveryMode.isSabr() || authenticatedInfo == null) { + return StreamResolution(publicResult) + } + val authenticatedResult = userId?.let { authenticatedInfo(it, url) } + if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) + return if (publicResult.requiresYoutubeSession()) { + StreamResolution( + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), + ) + } else { + StreamResolution(publicResult) + } +} + private fun StreamResponse.hasPlayableSource(): Boolean = videoStreams.isNotEmpty() || videoOnlyStreams.isNotEmpty() || diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 2de55302..2be09e73 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -31,8 +31,11 @@ internal object StreamExtractionErrorMapper { sanitize(error.message) ?: "This premiere has not started yet", "scheduled_premiere", ) + is AgeRestrictedContentException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: "This video is age-restricted", + "age_restricted", + ) is GeographicRestrictionException, - is AgeRestrictedContentException, is PrivateContentException -> ExtractionResult.BadRequest(sanitize(error.message) ?: "Content not available") else -> ExtractionResult.Failure( sanitize(error.message) ?: fallback, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt new file mode 100644 index 00000000..ae313935 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -0,0 +1,32 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" +internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" + +internal class YoutubeSessionSabrStreamService( + private val metadataService: YoutubeSessionStreamService, + private val infoService: AuthenticatedSabrInfoService, +) { + suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { + val metadata = metadataService.getStreamInfo(userId, url) ?: return null + if (metadata !is ExtractionResult.Success) return metadata + val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") + return when (val info = infoService.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> + ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) + AuthenticatedSabrInfoResult.Failed -> + ExtractionResult.Failure("Authenticated SABR playback unavailable") + AuthenticatedSabrInfoResult.NotConnected -> null + } + } +} + +internal fun ExtractionResult.requiresYoutubeSession(): Boolean = + when (this) { + is ExtractionResult.Success -> data.requiresMembership + is ExtractionResult.BadRequest -> code == "age_restricted" || code == "members_only" + is ExtractionResult.Failure -> false + } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index 0c14ab8c..d9ca60b6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -36,7 +36,10 @@ class YoutubeSessionStreamService( } if (requiresReconnect(result)) { youtubeSessionService.markNeedsReconnect(credentials.userId) - return ExtractionResult.BadRequest(YOUTUBE_SESSION_RECONNECT_ERROR) + return ExtractionResult.BadRequest( + YOUTUBE_SESSION_RECONNECT_ERROR, + YOUTUBE_SESSION_RECONNECT_CODE, + ) } youtubeSessionService.markUsed(credentials.userId) return result @@ -75,3 +78,5 @@ class YoutubeSessionStreamService( const val AUTHENTICATED_STREAM_MAX_TTL_SECONDS = 900L } } + +internal const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" From f8d9eb33d1e387ef75bdf6fef3db57122c54a469 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 14:03:35 +0200 Subject: [PATCH 33/65] test: cover gated YouTube playback access --- .../server/StreamExtractionErrorMapperTest.kt | 10 +++ .../dev/typetype/server/StreamRoutesTest.kt | 40 +++++++++++ .../server/YoutubeSessionStreamServiceTest.kt | 2 + .../routes/SabrPlaybackAccessValidatorTest.kt | 67 +++++++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index dc8c27a3..25db135c 100644 --- a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -7,6 +7,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException @@ -70,6 +71,15 @@ class StreamExtractionErrorMapperTest { assertEquals(ExtractionResult.BadRequest("private video"), result) } + @Test + fun `maps age restrictions to a stable access code`() { + val result = StreamExtractionErrorMapper.map(AgeRestrictedContentException("Sign in to confirm your age")) + assertEquals( + ExtractionResult.BadRequest("Sign in is required to verify access to this video", "age_restricted"), + result, + ) + } + @Test fun `maps unknown exceptions to failure`() { val result = StreamExtractionErrorMapper.map(IllegalStateException("boom")) diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt index 69794325..d9d6cac6 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -71,6 +71,46 @@ class StreamRoutesTest { assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) } + @Test + fun `GET restricted YouTube streams asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.BadRequest("Sign in to confirm your age", "age_restricted") + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=restricted") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + + @Test + fun `GET members-only metadata asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse().copy(requiresMembership = true)) + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=members") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + @Test fun `GET sabr streams returns 422 when final response has no playable source`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt index bcaa17d8..9049f2ff 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt @@ -11,6 +11,7 @@ import dev.typetype.server.services.YoutubeSessionCompleteResult import dev.typetype.server.services.YoutubeSessionCrypto import dev.typetype.server.services.YoutubeSessionService import dev.typetype.server.services.YoutubeSessionStreamService +import dev.typetype.server.services.YOUTUBE_SESSION_RECONNECT_CODE import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull @@ -70,6 +71,7 @@ class YoutubeSessionStreamServiceTest { val result = service.getStreamInfo(TEST_USER_ID, "https://youtube.com/watch?v=test") assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) assertEquals("needs_reconnect", youtubeSessionService.status(TEST_USER_ID).status) } diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt new file mode 100644 index 00000000..dd1e0683 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.StreamService +import dev.typetype.server.testStreamResponse +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SabrPlaybackAccessValidatorTest { + @Test + fun `uses linked YouTube session even when public metadata is accessible`() = runBlocking { + val authenticated = ExtractionResult.Success(testStreamResponse().copy(title = "Authenticated")) + val validator = validator( + publicResult = ExtractionResult.Success(testStreamResponse().copy(title = "Public")), + authenticatedResult = authenticated, + ) + + assertEquals(authenticated, validator.resolve("user-id", "video-id")) + } + + @Test + fun `uses linked YouTube session for age-restricted playback`() = runBlocking { + val authenticated = ExtractionResult.Success(testStreamResponse()) + val validator = validator( + publicResult = ExtractionResult.BadRequest("Confirm your age", "age_restricted"), + authenticatedResult = authenticated, + ) + + assertEquals(authenticated, validator.resolve("user-id", "video-id")) + } + + @Test + fun `asks for YouTube connection when restricted playback has no session`() = runBlocking { + val validator = validator( + publicResult = ExtractionResult.BadRequest("Sign in", "members_only"), + authenticatedResult = null, + ) + + assertEquals( + ExtractionResult.BadRequest("Connect YouTube to access this video", "youtube_session_required"), + validator.resolve(null, "video-id"), + ) + } + + @Test + fun `keeps membership error when linked account lacks access`() = runBlocking { + val membersOnly = ExtractionResult.BadRequest("Join this channel", "members_only") + val validator = validator( + publicResult = ExtractionResult.Success(testStreamResponse().copy(requiresMembership = true)), + authenticatedResult = membersOnly, + ) + + assertEquals(membersOnly, validator.resolve("user-id", "video-id")) + } + + private fun validator( + publicResult: ExtractionResult, + authenticatedResult: ExtractionResult?, + ): SabrPlaybackAccessValidator = SabrPlaybackAccessValidator( + publicStreamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult = publicResult + }, + youtubeSessionStreamInfo = { _, _ -> authenticatedResult }, + ) +} From b086e61384e21575a226d8c1e44e3258beae54d4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 14:03:35 +0200 Subject: [PATCH 34/65] docs: document YouTube session playback errors --- openapi/paths/streams.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index 33aad372..a0bab142 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -18,7 +18,11 @@ YoutubeSabrStreams: schema: $ref: ../components/streams.yaml#/StreamResponse '400': - $ref: ../components/common.yaml#/JsonError + description: Invalid request or YouTube account connection required (`youtube_session_required`). + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse '401': $ref: ../components/common.yaml#/JsonError '403': @@ -45,7 +49,11 @@ YoutubeSabrBootstrap: schema: $ref: ../components/streams.yaml#/StreamResponse '400': - $ref: ../components/common.yaml#/JsonError + description: Invalid request or YouTube account connection required (`youtube_session_required`). + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse '401': $ref: ../components/common.yaml#/JsonError '403': From 97afc7164f5f526cddf1d42b021a8bfc2fb17f85 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 18:16:03 +0200 Subject: [PATCH 35/65] fix: preserve authenticated YouTube player tokens --- .../server/services/NewPipeInitializer.kt | 4 ++- .../TypetypeYoutubeSessionPoTokenProvider.kt | 13 +++++++ ...petypeYoutubeSessionPoTokenProviderTest.kt | 36 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index 93ab24a8..b6cc04b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -17,10 +17,12 @@ object NewPipeInitializer { val normalizedUrl = tokenServiceUrl?.trim()?.takeIf { it.isNotBlank() } if (normalizedUrl != null && normalizedUrl != decoderServiceUrl) { YoutubeApiDecoder.setLocalDecoder(TypetypeTokenYoutubeJavaScriptDecoder(normalizedUrl)) - NewPipe.setYoutubeSessionPoTokenProvider( + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider( TypetypeTokenYoutubeSessionPoTokenProvider(normalizedUrl), ) decoderServiceUrl = normalizedUrl + } else if (normalizedUrl == null) { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) } if (!initialized) { NewPipe.init(OkHttpDownloader.instance(youtubeProxySelector)) diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index 7ef95956..74351ac3 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -7,6 +7,11 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvid internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { private val scopedToken = ThreadLocal() + @Volatile private var authenticatedProvider: YoutubeSessionPoTokenProvider? = null + + fun configureAuthenticatedProvider(provider: YoutubeSessionPoTokenProvider?): Unit { + authenticatedProvider = provider + } fun withToken(token: SabrTokenBundle, block: () -> T): T { return withToken(token.youtubeSessionPoToken(), block) @@ -30,4 +35,12 @@ internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenPro contentCountry: ContentCountry, loggedIn: Boolean, ): YoutubeSessionPoToken? = scopedToken.get() + ?: authenticatedProvider?.getSessionPoToken( + clientName, + clientVersion, + userAgent, + localization, + contentCountry, + loggedIn, + ) } diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt index 7d80673c..3a6a12e4 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt @@ -2,11 +2,18 @@ package dev.typetype.server.services import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider class TypetypeYoutubeSessionPoTokenProviderTest { + @AfterEach + fun clearAuthenticatedProvider(): Unit = + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) + @Test fun `exposes the session token only inside its scope`() { TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { @@ -40,6 +47,24 @@ class TypetypeYoutubeSessionPoTokenProviderTest { assertNull(currentToken()) } + @Test + fun `uses the authenticated provider outside a SABR scope`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + assertEquals("auth", currentToken()?.visitorData) + assertEquals("auth-token", currentToken()?.poToken) + } + + @Test + fun `prefers the SABR token over the authenticated provider`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { + assertEquals("sabr", currentToken()?.visitorData) + assertEquals("sabr-token", currentToken()?.poToken) + } + } + private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( "MWEB", "2.20260801.00.00", @@ -57,4 +82,15 @@ class TypetypeYoutubeSessionPoTokenProviderTest { videoBoundPoToken = "video-token", videoBoundPoTokenBytes = byteArrayOf(2), ) + + private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { + override fun getSessionPoToken( + clientName: String, + clientVersion: String, + userAgent: String?, + localization: Localization, + contentCountry: ContentCountry, + loggedIn: Boolean, + ) = YoutubeSessionPoToken(visitorData, poToken) + } } From eb2e206c10c6580162f89fbf1ac9c7d2b8a8ed80 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 15 Aug 2026 18:16:12 +0200 Subject: [PATCH 36/65] fix: preserve selected YouTube account --- openapi/components/youtube-session.yaml | 3 +- openapi/paths/youtube-session.yaml | 3 +- .../server/ExtractionServiceRegistry.kt | 5 +- .../dev/typetype/server/db/DatabaseFactory.kt | 1 + .../server/db/tables/YoutubeSessionsTable.kt | 1 + .../server/downloader/OkHttpDownloader.kt | 7 ++- .../downloader/YoutubeAuthUserContext.kt | 20 +++++++ .../YoutubeRemoteBrowserCompleteRequest.kt | 1 + .../models/YoutubeSessionCompleteRequest.kt | 1 + .../services/YoutubeRemoteBrowserService.kt | 9 +++- .../services/YoutubeSessionCredentials.kt | 1 + .../server/services/YoutubeSessionService.kt | 32 ++++++++--- .../server/services/YoutubeSessionStore.kt | 54 +++++++++++++++---- .../services/YoutubeSessionTokenScope.kt | 4 ++ .../server/OkHttpDownloaderCoreTest.kt | 13 +++++ ...YoutubeAuthenticatedExtractionProbeTest.kt | 8 +-- .../YoutubeRemoteBrowserCompleteRoutesTest.kt | 11 +++- .../server/YoutubeSessionRoutesTest.kt | 16 ++++-- .../server/YoutubeSessionTokenScopeTest.kt | 10 +++- 19 files changed, 168 insertions(+), 32 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt diff --git a/openapi/components/youtube-session.yaml b/openapi/components/youtube-session.yaml index 8ce67139..963d9ab3 100644 --- a/openapi/components/youtube-session.yaml +++ b/openapi/components/youtube-session.yaml @@ -14,7 +14,7 @@ YoutubeRemoteBrowserStartResponse: expiresAt: { type: integer, format: int64 } YoutubeRemoteBrowserCompleteRequest: type: object - required: [sessionId, tokenSessionId, status, cookies, poToken, capturedAt] + required: [sessionId, tokenSessionId, status, cookies, poToken, authUser, capturedAt] properties: sessionId: { type: string } tokenSessionId: { type: string } @@ -23,4 +23,5 @@ YoutubeRemoteBrowserCompleteRequest: enum: [completed] cookies: { type: string } poToken: { type: string } + authUser: { type: integer, minimum: 0, maximum: 99 } capturedAt: { type: integer, format: int64 } diff --git a/openapi/paths/youtube-session.yaml b/openapi/paths/youtube-session.yaml index 2e22552b..a491b069 100644 --- a/openapi/paths/youtube-session.yaml +++ b/openapi/paths/youtube-session.yaml @@ -69,7 +69,7 @@ InternalBrowserComplete: application/json: schema: type: object - required: [sessionId, tokenSessionId, status, cookies, poToken, capturedAt] + required: [sessionId, tokenSessionId, status, cookies, poToken, authUser, capturedAt] properties: sessionId: { type: string } tokenSessionId: { type: string } @@ -78,6 +78,7 @@ InternalBrowserComplete: enum: [completed] cookies: { type: string } poToken: { type: string } + authUser: { type: integer, minimum: 0, maximum: 99 } capturedAt: { type: integer, format: int64 } responses: '204': { description: YouTube credentials stored. } diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 553531cc..5791d986 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -42,7 +42,6 @@ import dev.typetype.server.services.OkHttpYouTubeSubtitleContentFetcher import dev.typetype.server.services.StreamYouTubeSubtitleResolver import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient -import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService import dev.typetype.server.services.YoutubePlayerClientStreamService import dev.typetype.server.services.YoutubeScopedChannelService import dev.typetype.server.services.YoutubeScopedCommentService @@ -98,9 +97,9 @@ internal class ExtractionServiceRegistry( directPipePipeStreamService, YoutubePlayerClient.VISIONOS, ) - private val authenticatedStreamService = YoutubePlayerClientFallbackStreamService( + private val authenticatedStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), + YoutubePlayerClient.MWEB, ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( sabrPipePipeStreamService, diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index f964f6db..0da240a5 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -119,6 +119,7 @@ object DatabaseFactory { exec("ALTER TABLE users ADD COLUMN IF NOT EXISTS public_username TEXT") exec("ALTER TABLE users ADD COLUMN IF NOT EXISTS bio TEXT") exec("ALTER TABLE youtube_takeout_import_jobs ADD COLUMN IF NOT EXISTS preview_json TEXT") + exec("ALTER TABLE youtube_sessions ADD COLUMN IF NOT EXISTS auth_user INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE bug_reports ALTER COLUMN github_issue_url TYPE TEXT") DatabaseSessionAuthMigration.apply() DatabaseOidcMigration.apply() diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt index a904aadb..3d05bf41 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt @@ -6,6 +6,7 @@ object YoutubeSessionsTable : Table("youtube_sessions") { val userId = text("user_id") val encryptedCookies = text("encrypted_cookies") val encryptedPoToken = text("encrypted_po_token") + val authUser = integer("auth_user").default(0) val status = text("status") val createdAt = long("created_at") val updatedAt = long("updated_at") diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt index de00fc06..1e9072ca 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt +++ b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt @@ -40,6 +40,7 @@ class OkHttpDownloader private constructor( } private const val STREAMING_READ_TIMEOUT_MS = 30_000L + private const val YOUTUBE_AUTH_USER_HEADER = "X-Goog-AuthUser" } override fun execute(request: ExtractorRequest): Response { @@ -102,15 +103,19 @@ class OkHttpDownloader private constructor( private fun buildOkHttpRequest(request: ExtractorRequest): Request { val method = request.httpMethod() val dataToSend = request.dataToSend() + val normalizedUrl = normalizeExtractorUrl(request.url()) val body = dataToSend?.toRequestBody() ?: if (method == "POST" || method == "PUT" || method == "PATCH") ByteArray(0).toRequestBody() else null val builder = Request.Builder() - .url(normalizeExtractorUrl(request.url())) + .url(normalizedUrl) .method(method, body) request.headers().forEach { (name, values) -> values.forEach { value -> builder.addHeader(name, value) } } + YoutubeAuthUserContext.headerFor(normalizedUrl)?.let { + builder.header(YOUTUBE_AUTH_USER_HEADER, it) + } return builder.build() } diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt b/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt new file mode 100644 index 00000000..ad03ff60 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.downloader + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +internal object YoutubeAuthUserContext { + @Volatile private var value: Int? = null + + fun set(authUser: Int?): Unit { + value = authUser + } + + internal fun headerFor(url: String): String? { + val parsed = url.toHttpUrlOrNull() ?: return null + val host = parsed.host.lowercase() + val isYoutube = host == "youtube.com" || host.endsWith(".youtube.com") + return value?.toString()?.takeIf { + isYoutube && parsed.encodedPath.startsWith("/youtubei/v1/") + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt b/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt index 42534723..f45ee274 100644 --- a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt @@ -9,5 +9,6 @@ data class YoutubeRemoteBrowserCompleteRequest( val status: String, val cookies: String, val poToken: String, + val authUser: Int = 0, val capturedAt: Long, ) diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt b/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt index 261033fc..50855e4b 100644 --- a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt @@ -7,4 +7,5 @@ data class YoutubeSessionCompleteRequest( val code: String, val cookies: String, val poToken: String, + val authUser: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt index b4ae2798..a5d6d429 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt @@ -45,7 +45,14 @@ class YoutubeRemoteBrowserService( if (!youtubeSessionService.isConfigured) return YoutubeRemoteBrowserCompleteResult.Unavailable val session = sessions.complete(request.sessionId, request.tokenSessionId) ?: return YoutubeRemoteBrowserCompleteResult.NotFound - return when (youtubeSessionService.completeRemote(session.userId, request.cookies, request.poToken)) { + return when ( + youtubeSessionService.completeRemote( + session.userId, + request.cookies, + request.poToken, + request.authUser, + ) + ) { YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed YoutubeSessionCompleteResult.InvalidCode, YoutubeSessionCompleteResult.ExpiredCode, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt index 510200d5..111a1acc 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt @@ -5,4 +5,5 @@ data class YoutubeSessionCredentials( val fingerprint: String, val cookies: String, val poToken: String, + val authUser: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt index 43dc4a8e..0575a407 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt @@ -20,28 +20,35 @@ class YoutubeSessionService( val cookies = YoutubeSessionCookieNormalizer.normalize(request.cookies) ?: return YoutubeSessionCompleteResult.InvalidCredentials val poToken = request.poToken.trim() - if (code.isBlank() || !YoutubeSessionCredentialValidator.isValid(cookies, poToken)) { + if (code.isBlank() || !validCredentials(cookies, poToken, request.authUser)) { return YoutubeSessionCompleteResult.InvalidCredentials } return store.complete( code = code, encryptedCookies = crypto.encrypt(cookies), encryptedPoToken = crypto.encrypt(poToken), + authUser = request.authUser, ) } - suspend fun completeRemote(userId: String, rawCookies: String, rawPoToken: String): YoutubeSessionCompleteResult { + suspend fun completeRemote( + userId: String, + rawCookies: String, + rawPoToken: String, + authUser: Int = 0, + ): YoutubeSessionCompleteResult { val crypto = crypto ?: return YoutubeSessionCompleteResult.Unavailable val cookies = YoutubeSessionCookieNormalizer.normalize(rawCookies) ?: return YoutubeSessionCompleteResult.InvalidCredentials val poToken = rawPoToken.trim() - if (!YoutubeSessionCredentialValidator.isValid(cookies, poToken)) { + if (!validCredentials(cookies, poToken, authUser)) { return YoutubeSessionCompleteResult.InvalidCredentials } store.completeForUser( userId = userId, encryptedCookies = crypto.encrypt(cookies), encryptedPoToken = crypto.encrypt(poToken), + authUser = authUser, ) return YoutubeSessionCompleteResult.Completed } @@ -57,9 +64,15 @@ class YoutubeSessionService( val credentials = runCatching { YoutubeSessionCredentials( userId = userId, - fingerprint = PublicCacheKey.of("youtube-session", encrypted.first, encrypted.second), - cookies = crypto.decrypt(encrypted.first), - poToken = crypto.decrypt(encrypted.second), + fingerprint = PublicCacheKey.of( + "youtube-session", + encrypted.cookies, + encrypted.poToken, + encrypted.authUser.toString(), + ), + cookies = crypto.decrypt(encrypted.cookies), + poToken = crypto.decrypt(encrypted.poToken), + authUser = encrypted.authUser, ) }.getOrNull() if (credentials == null) store.markNeedsReconnect(userId) @@ -69,4 +82,11 @@ class YoutubeSessionService( suspend fun markUsed(userId: String): Unit = store.markUsed(userId) suspend fun markNeedsReconnect(userId: String): Unit = store.markNeedsReconnect(userId) + + private fun validCredentials(cookies: String, poToken: String, authUser: Int): Boolean = + authUser in 0..MAX_AUTH_USER && YoutubeSessionCredentialValidator.isValid(cookies, poToken) + + private companion object { + const val MAX_AUTH_USER = 99 + } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt index 2d424534..8e7f7f32 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt @@ -13,7 +13,12 @@ import org.jetbrains.exposed.v1.jdbc.update class YoutubeSessionStore( private val nowMillis: () -> Long = System::currentTimeMillis, ) { - suspend fun complete(code: String, encryptedCookies: String, encryptedPoToken: String): YoutubeSessionCompleteResult { + suspend fun complete( + code: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + ): YoutubeSessionCompleteResult { val now = nowMillis() return DatabaseFactory.query { val pairing = YoutubeSessionPairingsTable.selectAll() @@ -23,7 +28,7 @@ class YoutubeSessionStore( YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.code eq code } return@query YoutubeSessionCompleteResult.ExpiredCode } - upsertSession(pairing[YoutubeSessionPairingsTable.userId], encryptedCookies, encryptedPoToken, now) + upsertSession(pairing[YoutubeSessionPairingsTable.userId], encryptedCookies, encryptedPoToken, authUser, now) YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.code eq code } YoutubeSessionCompleteResult.Completed } @@ -45,17 +50,28 @@ class YoutubeSessionStore( YoutubeSessionsTable.deleteWhere { YoutubeSessionsTable.userId eq userId } > 0 } - suspend fun completeForUser(userId: String, encryptedCookies: String, encryptedPoToken: String): Unit = + suspend fun completeForUser( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + ): Unit = DatabaseFactory.query { - upsertSession(userId, encryptedCookies, encryptedPoToken, nowMillis()) + upsertSession(userId, encryptedCookies, encryptedPoToken, authUser, nowMillis()) } - suspend fun connectedEncrypted(userId: String): Pair? = DatabaseFactory.query { + suspend fun connectedEncrypted(userId: String): EncryptedYoutubeSessionCredentials? = DatabaseFactory.query { YoutubeSessionsTable.selectAll() .where { YoutubeSessionsTable.userId eq userId } .singleOrNull() ?.takeIf { YoutubeSessionStatus.from(it[YoutubeSessionsTable.status]) == YoutubeSessionStatus.Connected } - ?.let { it[YoutubeSessionsTable.encryptedCookies] to it[YoutubeSessionsTable.encryptedPoToken] } + ?.let { + EncryptedYoutubeSessionCredentials( + cookies = it[YoutubeSessionsTable.encryptedCookies], + poToken = it[YoutubeSessionsTable.encryptedPoToken], + authUser = it[YoutubeSessionsTable.authUser], + ) + } } suspend fun markUsed(userId: String): Unit = DatabaseFactory.query { @@ -71,22 +87,36 @@ class YoutubeSessionStore( } } - private fun upsertSession(userId: String, encryptedCookies: String, encryptedPoToken: String, now: Long) { + private fun upsertSession( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + now: Long, + ) { val updated = YoutubeSessionsTable.update({ YoutubeSessionsTable.userId eq userId }) { it[YoutubeSessionsTable.encryptedCookies] = encryptedCookies it[YoutubeSessionsTable.encryptedPoToken] = encryptedPoToken + it[YoutubeSessionsTable.authUser] = authUser it[status] = YoutubeSessionStatus.Connected.value it[updatedAt] = now it[lastUsedAt] = 0 } - if (updated == 0) insertSession(userId, encryptedCookies, encryptedPoToken, now) + if (updated == 0) insertSession(userId, encryptedCookies, encryptedPoToken, authUser, now) } - private fun insertSession(userId: String, encryptedCookies: String, encryptedPoToken: String, now: Long) { + private fun insertSession( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + now: Long, + ) { YoutubeSessionsTable.insert { it[YoutubeSessionsTable.userId] = userId it[YoutubeSessionsTable.encryptedCookies] = encryptedCookies it[YoutubeSessionsTable.encryptedPoToken] = encryptedPoToken + it[YoutubeSessionsTable.authUser] = authUser it[status] = YoutubeSessionStatus.Connected.value it[createdAt] = now it[updatedAt] = now @@ -94,3 +124,9 @@ class YoutubeSessionStore( } } } + +data class EncryptedYoutubeSessionCredentials( + val cookies: String, + val poToken: String, + val authUser: Int, +) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt index b6f45dcf..5c0c3c6f 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.downloader.YoutubeAuthUserContext import org.schabi.newpipe.extractor.ServiceList import java.util.concurrent.Semaphore import kotlinx.coroutines.Dispatchers @@ -13,18 +14,21 @@ object YoutubeSessionTokenScope { withPermits(PUBLIC_PERMITS) { val youtube = ServiceList.YouTube try { + YoutubeAuthUserContext.set(credentials.authUser) youtube.setTokens(credentials.cookies) youtube.setAdditionalTokens(credentials.poToken) block() } finally { youtube.setTokens("") youtube.setAdditionalTokens("") + YoutubeAuthUserContext.set(null) } } suspend fun withoutCredentials(block: suspend () -> T): T = withPermits(1) { val youtube = ServiceList.YouTube + YoutubeAuthUserContext.set(null) youtube.setTokens("") youtube.setAdditionalTokens("") block() diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt b/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt index 38a6dca9..28df1516 100644 --- a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt @@ -2,6 +2,7 @@ package dev.typetype.server import com.sun.net.httpserver.HttpServer import dev.typetype.server.downloader.OkHttpDownloader +import dev.typetype.server.downloader.YoutubeAuthUserContext import dev.typetype.server.downloader.normalizeExtractorUrl import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull @@ -16,6 +17,18 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit class OkHttpDownloaderCoreTest { + @Test + fun `YouTube auth user is limited to InnerTube requests`() { + YoutubeAuthUserContext.set(3) + try { + assertEquals("3", YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player")) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://www.youtube.com/watch?v=test")) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://example.com/youtubei/v1/player")) + } finally { + YoutubeAuthUserContext.set(null) + } + } + @Test fun `execute maps http response payload`() { val server = server(status = 200, body = "ok") diff --git a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt index 0f01f486..e74fd18f 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt @@ -11,7 +11,7 @@ import dev.typetype.server.services.TypetypeTokenSabrTokenClient import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient import dev.typetype.server.services.YouTubeSubtitleService import dev.typetype.server.services.YoutubePlayerClient -import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService +import dev.typetype.server.services.YoutubePlayerClientStreamService import dev.typetype.server.services.YoutubeSessionCookieNormalizer import dev.typetype.server.services.YoutubeSessionCredentials import dev.typetype.server.services.YoutubeSessionTokenScope @@ -45,15 +45,15 @@ class YoutubeAuthenticatedExtractionProbeTest { fun `authenticated classic extraction receives a session bound player token`() = runBlocking { NewPipeInitializer.init(tokenServiceUrl) val cookies = readCookies() - val credentials = YoutubeSessionCredentials("probe", "probe", cookies, "probe-token") + val credentials = YoutubeSessionCredentials("probe", "probe", cookies, "probe-token", authUser = 1) val pipePipe = PipePipeStreamService( ProbeCache, YouTubeSubtitleService(OkHttpClient(), tokenServiceUrl), BilibiliRelatedService(), ) - val service = YoutubePlayerClientFallbackStreamService( + val service = YoutubePlayerClientStreamService( pipePipe, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), + YoutubePlayerClient.MWEB, ) val result = YoutubeSessionTokenScope.withCredentials(credentials) { diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt index 8e55fda6..82f5413f 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt @@ -102,14 +102,21 @@ class YoutubeRemoteBrowserCompleteRoutesTest { } private fun completeBody(sessionId: String): String = - """{"sessionId":"$sessionId","tokenSessionId":"token-session","status":"completed","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","capturedAt":123}""" + """{"sessionId":"$sessionId","tokenSessionId":"token-session","status":"completed","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","authUser":2,"capturedAt":123}""" private suspend fun assertCredentialsAreEncrypted() { val encrypted = DatabaseFactory.query { YoutubeSessionsTable.selectAll().where { YoutubeSessionsTable.userId eq TEST_USER_ID }.single() - .let { it[YoutubeSessionsTable.encryptedCookies] to it[YoutubeSessionsTable.encryptedPoToken] } + .let { + Triple( + it[YoutubeSessionsTable.encryptedCookies], + it[YoutubeSessionsTable.encryptedPoToken], + it[YoutubeSessionsTable.authUser], + ) + } } assertFalse(encrypted.first.contains("secret-cookie")) assertFalse(encrypted.second.contains("secret-pot")) + assertEquals(2, encrypted.third) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt index 0f679437..1299da41 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt @@ -67,10 +67,10 @@ class YoutubeSessionRoutesTest { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText()).code - private suspend fun ApplicationTestBuilder.completeSession(code: String) = client.post("/youtube-session/complete") { + private suspend fun ApplicationTestBuilder.completeSession(code: String, authUser: Int = 2) = client.post("/youtube-session/complete") { headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) setBody( - """{"code":"$code","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value"}""", + """{"code":"$code","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","authUser":$authUser}""", ) } @@ -110,12 +110,22 @@ class YoutubeSessionRoutesTest { assertEquals(HttpStatusCode.Gone, completeSession(code).status) } + @Test + fun `complete rejects an invalid Google account index`() = withApp { + assertEquals(HttpStatusCode.BadRequest, completeSession(pairingCode(), authUser = 100).status) + } + private suspend fun assertCredentialsAreEncrypted() { val encrypted = DatabaseFactory.query { val row = YoutubeSessionsTable.selectAll().where { YoutubeSessionsTable.userId eq TEST_USER_ID }.single() - row[YoutubeSessionsTable.encryptedCookies] to row[YoutubeSessionsTable.encryptedPoToken] + Triple( + row[YoutubeSessionsTable.encryptedCookies], + row[YoutubeSessionsTable.encryptedPoToken], + row[YoutubeSessionsTable.authUser], + ) } assertFalse(encrypted.first.contains("secret-cookie")) assertFalse(encrypted.second.contains("secret-pot")) + assertEquals(2, encrypted.third) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt index 10da62ed..a7e06bc0 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt @@ -1,5 +1,6 @@ package dev.typetype.server +import dev.typetype.server.downloader.YoutubeAuthUserContext import dev.typetype.server.services.YoutubeSessionCredentials import dev.typetype.server.services.YoutubeSessionTokenScope import kotlinx.coroutines.runBlocking @@ -19,14 +20,21 @@ class YoutubeSessionTokenScopeTest { fingerprint = "session-fingerprint", cookies = "SID=session-cookie", poToken = "session-pot-value", + authUser = 2, ) ) { - youtube.tokens to youtube.additionalTokens + Triple( + youtube.tokens, + youtube.additionalTokens, + YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player"), + ) } assertEquals("SID=session-cookie", observed.first) assertEquals("session-pot-value", observed.second) + assertEquals("2", observed.third) assertEquals("", youtube.tokens.orEmpty()) assertEquals("", youtube.additionalTokens.orEmpty()) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player")) } @Test From 7c9c93b4c6bec596487f999fc4c0e3285067358a Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:34:25 -0700 Subject: [PATCH 37/65] feat: persist subscription group records Add account-scoped group and membership tables plus the API models used by the group service. Register both tables in production and test database setup. --- .../dev/typetype/server/db/DatabaseFactory.kt | 4 ++++ .../SubscriptionGroupMembershipsTable.kt | 17 +++++++++++++++++ .../db/tables/SubscriptionGroupsTable.kt | 18 ++++++++++++++++++ .../server/models/SubscriptionGroupItem.kt | 12 ++++++++++++ .../SubscriptionGroupMembershipRequest.kt | 6 ++++++ .../server/models/SubscriptionGroupRequest.kt | 6 ++++++ .../services/SubscriptionGroupResults.kt | 17 +++++++++++++++++ .../kotlin/dev/typetype/server/TestDatabase.kt | 4 ++++ 8 files changed, 84 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 0da240a5..4d2eeb7e 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -16,6 +16,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.UserAvatarsTable import dev.typetype.server.db.tables.WatchLaterTable @@ -57,6 +59,8 @@ object DatabaseFactory { AdminSettingsTable, HistoryTable, SubscriptionsTable, + SubscriptionGroupsTable, + SubscriptionGroupMembershipsTable, PlaylistsTable, PlaylistVideosTable, WatchLaterTable, diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt new file mode 100644 index 00000000..93931a89 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.ReferenceOption +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupMembershipsTable : Table("subscription_group_memberships") { + val groupId = text("group_id").references(SubscriptionGroupsTable.id, onDelete = ReferenceOption.CASCADE) + val userId = text("user_id") + val channelUrl = text("channel_url") + val addedAt = long("added_at") + + init { + index(false, userId, channelUrl) + } + + override val primaryKey = PrimaryKey(groupId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt new file mode 100644 index 00000000..13b89b20 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupsTable : Table("subscription_groups") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val normalizedName = text("normalized_name") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + uniqueIndex(userId, normalizedName) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt new file mode 100644 index 00000000..684382bb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupItem( + val id: String, + val name: String, + val channelCount: Int, + val createdAt: Long, + val updatedAt: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt new file mode 100644 index 00000000..9a2fcb5f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupMembershipRequest(val channelUrl: String) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt new file mode 100644 index 00000000..136b831c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupRequest(val name: String) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt new file mode 100644 index 00000000..6c19f560 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubscriptionGroupItem + +sealed interface SubscriptionGroupWriteResult { + data class Success(val group: SubscriptionGroupItem) : SubscriptionGroupWriteResult + data object InvalidName : SubscriptionGroupWriteResult + data object DuplicateName : SubscriptionGroupWriteResult + data object NotFound : SubscriptionGroupWriteResult +} + +sealed interface SubscriptionGroupMembershipResult { + data object Success : SubscriptionGroupMembershipResult + data object GroupNotFound : SubscriptionGroupMembershipResult + data object SubscriptionNotFound : SubscriptionGroupMembershipResult + data object MembershipNotFound : SubscriptionGroupMembershipResult +} diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index 1a22cc72..be1dc356 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -24,6 +24,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -102,6 +104,8 @@ object TestDatabase { HistoryTable.deleteAll() FavoritesTable.deleteAll() SettingsTable.deleteAll() + SubscriptionGroupMembershipsTable.deleteAll() + SubscriptionGroupsTable.deleteAll() SubscriptionsTable.deleteAll() WatchLaterTable.deleteAll() ProgressTable.deleteAll() From 7ef9808753ce7cf464dd4f2336464b7e5f598083 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:34:33 -0700 Subject: [PATCH 38/65] feat: manage subscription group membership Create, rename, and remove account-owned groups and assign subscribed channels to multiple groups. Keep memberships consistent when subscriptions are deleted or replaced by imports. --- .../PipePipeBackupPersisterService.kt | 1 + .../SubscriptionGroupMembershipCleaner.kt | 19 ++ .../services/SubscriptionGroupsService.kt | 182 ++++++++++++++++++ .../server/services/SubscriptionSelection.kt | 17 ++ .../server/services/SubscriptionsService.kt | 36 +++- .../services/TypeTypeBackupCoreRestore.kt | 1 + 6 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt index 9378fb54..4e965014 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt @@ -22,6 +22,7 @@ class PipePipeBackupPersisterService { .toMap() val history = insertHistory(userId, snapshot.history, avatarsByChannel) val subscriptions = insertSubscriptions(userId, snapshot.subscriptions) + SubscriptionGroupMembershipCleaner.retain(userId, snapshot.subscriptions.map { it.url }) val (playlists, playlistVideos) = insertPlaylists(userId, snapshot.playlists) val progress = insertProgress(userId, snapshot.progress) val searchHistory = insertSearchHistory(userId, snapshot.searchHistory) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt new file mode 100644 index 00000000..d78f7833 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.notInList +import org.jetbrains.exposed.v1.jdbc.deleteWhere + +internal object SubscriptionGroupMembershipCleaner { + fun retain(userId: String, channelUrls: Collection) { + val retained = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + SubscriptionGroupMembershipsTable.deleteWhere { + val ownedByUser = SubscriptionGroupMembershipsTable.userId eq userId + if (retained.isEmpty()) ownedByUser else { + ownedByUser and (SubscriptionGroupMembershipsTable.channelUrl notInList retained) + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt new file mode 100644 index 00000000..23887b2c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -0,0 +1,182 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupItem +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import java.sql.SQLException +import java.util.Locale +import java.util.UUID + +class SubscriptionGroupsService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + val counts = SubscriptionGroupMembershipsTable.selectAll() + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .groupingBy { it[SubscriptionGroupMembershipsTable.groupId] } + .eachCount() + SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) + .map { it.toItem(counts[it[SubscriptionGroupsTable.id]] ?: 0) } + } + + suspend fun exists(userId: String, groupId: String): Boolean = DatabaseFactory.query { + groupExists(userId, groupId) + } + + suspend fun create(userId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return DatabaseFactory.query { + if (nameExists(userId, normalizedName)) return@query SubscriptionGroupWriteResult.DuplicateName + val id = UUID.randomUUID().toString() + val now = System.currentTimeMillis() + val inserted = SubscriptionGroupsTable.insertIgnore { + it[SubscriptionGroupsTable.id] = id + it[SubscriptionGroupsTable.userId] = userId + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[createdAt] = now + it[updatedAt] = now + }.insertedCount + if (inserted == 0) SubscriptionGroupWriteResult.DuplicateName else { + SubscriptionGroupWriteResult.Success(SubscriptionGroupItem(id, name, 0, now, now)) + } + } + } + + suspend fun rename(userId: String, groupId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return try { + DatabaseFactory.query { + val current = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.singleOrNull() ?: return@query SubscriptionGroupWriteResult.NotFound + val duplicate = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any { it[SubscriptionGroupsTable.id] != groupId } + if (duplicate) return@query SubscriptionGroupWriteResult.DuplicateName + val now = System.currentTimeMillis() + SubscriptionGroupsTable.update({ + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }) { + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[updatedAt] = now + } + val count = membershipCount(userId, groupId) + SubscriptionGroupWriteResult.Success( + SubscriptionGroupItem(groupId, name, count, current[SubscriptionGroupsTable.createdAt], now), + ) + } + } catch (error: Throwable) { + if (error.isUniqueConstraintViolation()) SubscriptionGroupWriteResult.DuplicateName else throw error + } + } + + suspend fun delete(userId: String, groupId: String): Boolean = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query false + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + } + SubscriptionGroupsTable.deleteWhere { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + } > 0 + } + + suspend fun addSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val subscriptionExists = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.any() + if (!subscriptionExists) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound + SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = System.currentTimeMillis() + } + SubscriptionGroupMembershipResult.Success + } + + suspend fun removeSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val deleted = SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq channelUrl) + } + if (deleted > 0) SubscriptionGroupMembershipResult.Success else { + SubscriptionGroupMembershipResult.MembershipNotFound + } + } + + suspend fun getChannelUrls(userId: String, groupId: String): List = DatabaseFactory.query { + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + }.orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.DESC) + .map { it[SubscriptionGroupMembershipsTable.channelUrl] } + } + + private fun groupExists(userId: String, groupId: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.any() + + private fun nameExists(userId: String, normalizedName: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any() + + private fun membershipCount(userId: String, groupId: String): Int = + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq groupId) + }.count().toInt() + + private fun ResultRow.toItem(channelCount: Int): SubscriptionGroupItem = SubscriptionGroupItem( + id = this[SubscriptionGroupsTable.id], + name = this[SubscriptionGroupsTable.name], + channelCount = channelCount, + createdAt = this[SubscriptionGroupsTable.createdAt], + updatedAt = this[SubscriptionGroupsTable.updatedAt], + ) + + private fun normalizeDisplayName(value: String): String? = + value.trim().takeIf { it.length in 1..MAX_GROUP_NAME_LENGTH } + + private fun normalizeUniqueName(value: String): String = value.lowercase(Locale.ROOT) + + private fun Throwable.isUniqueConstraintViolation(): Boolean = generateSequence(this) { it.cause } + .filterIsInstance() + .any { it.sqlState == UNIQUE_VIOLATION_SQL_STATE } + + companion object { + const val MAX_GROUP_NAME_LENGTH = 100 + private const val UNIQUE_VIOLATION_SQL_STATE = "23505" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt new file mode 100644 index 00000000..cbdf5d99 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +sealed interface SubscriptionSelection { + val cursorKey: String + + data object All : SubscriptionSelection { + override val cursorKey: String = "all" + } + + data object Ungrouped : SubscriptionSelection { + override val cursorKey: String = "ungrouped" + } + + data class Group(val id: String) : SubscriptionSelection { + override val cursorKey: String = "group:$id" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 887a931e..21d022de 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.SubscriptionItem import org.jetbrains.exposed.v1.core.ResultRow @@ -13,14 +14,22 @@ import org.jetbrains.exposed.v1.jdbc.selectAll class SubscriptionsService { - suspend fun getAll(userId: String): List = DatabaseFactory.query { + suspend fun getAll( + userId: String, + selection: SubscriptionSelection = SubscriptionSelection.All, + ): List = DatabaseFactory.query { + val selectedUrls = selectedChannelUrls(userId, selection) val items = SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) .map { it.toItem() } + .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } SubscriptionAvatarRepairer.repair(userId = userId, items = items) } + suspend fun getChannelUrls(userId: String, selection: SubscriptionSelection): Set = + DatabaseFactory.query { selectedChannelUrls(userId, selection) } + suspend fun add(userId: String, item: SubscriptionItem): SubscriptionItem { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) val now = System.currentTimeMillis() @@ -38,9 +47,34 @@ class SubscriptionsService { suspend fun delete(userId: String, channelUrl: String): Boolean = DatabaseFactory.query { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(channelUrl) + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq canonicalUrl) + } SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0 } + private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { + val all = SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq userId } + .mapTo(linkedSetOf()) { ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) } + if (selection == SubscriptionSelection.All) return all + val memberships = SubscriptionGroupMembershipsTable.selectAll().where { + when (selection) { + SubscriptionSelection.All -> SubscriptionGroupMembershipsTable.userId eq userId + SubscriptionSelection.Ungrouped -> SubscriptionGroupMembershipsTable.userId eq userId + is SubscriptionSelection.Group -> + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq selection.id) + } + }.mapTo(mutableSetOf()) { it[SubscriptionGroupMembershipsTable.channelUrl] } + return when (selection) { + SubscriptionSelection.All -> all + SubscriptionSelection.Ungrouped -> all - memberships + is SubscriptionSelection.Group -> all intersect memberships + } + } + private fun ResultRow.toItem() = SubscriptionItem( channelUrl = ChannelUrlCanonicalizer.canonicalize(this[SubscriptionsTable.channelUrl]), name = this[SubscriptionsTable.name], diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt index f8f5b715..7106218c 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt @@ -22,6 +22,7 @@ internal object TypeTypeBackupCoreRestore { this[SubscriptionsTable.avatarUrl] = item.avatarUrl this[SubscriptionsTable.subscribedAt] = item.subscribedAt } + SubscriptionGroupMembershipCleaner.retain(userId, items.map(SubscriptionItem::channelUrl)) return items.size } From 61b01ade3729463f1d266f23e377b090919e40fe Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:34:38 -0700 Subject: [PATCH 39/65] feat: expose subscription group API Add authenticated CRUD and membership endpoints backed by the account-scoped group service, and register the service with the application. --- .../dev/typetype/server/ServiceRegistry.kt | 2 + .../server/routes/SubscriptionGroupsRoutes.kt | 114 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 3164b3d1..de8e6a0b 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -29,6 +29,7 @@ import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsBlendService import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator import dev.typetype.server.services.TypeTypeBackupService @@ -84,6 +85,7 @@ internal class ServiceRegistry( val sabrSessionStore = extraction.sabrSessionStore val historyService = HistoryService() val subscriptionsService = SubscriptionsService() + val subscriptionGroupsService = SubscriptionGroupsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) val subscriptionShortsFeedService = SubscriptionShortsFeedService( subscriptionsService, diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt new file mode 100644 index 00000000..c80acb3d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -0,0 +1,114 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionGroupMembershipRequest +import dev.typetype.server.models.SubscriptionGroupRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, authService: AuthService) { + get("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> call.respond(groupsService.getAll(userId)) } + } + post("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.create(userId, request.name), created = true) + } + } + put("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.rename(userId, groupId, request.name), created = false) + } + } + delete("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + if (groupsService.delete(userId, groupId)) call.respond(HttpStatusCode.NoContent) else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found")) + } + } + } + put("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (request.channelUrl.isBlank()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("channelUrl must not be blank")) + } + call.respondMembership(groupsService.addSubscription(userId, groupId, request.channelUrl)) + } + } + delete("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val channelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) + call.respondMembership(groupsService.removeSubscription(userId, groupId, channelUrl)) + } + } +} + +private fun ApplicationCall.groupId(): String? = parameters["groupId"]?.takeIf(String::isNotBlank) + +private suspend fun ApplicationCall.receiveGroupRequest(): SubscriptionGroupRequest? = + runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + null + } + +private suspend fun ApplicationCall.respondGroupWrite(result: SubscriptionGroupWriteResult, created: Boolean) { + when (result) { + is SubscriptionGroupWriteResult.Success -> if (created) respond(HttpStatusCode.Created, result.group) else { + respond(HttpStatusCode.NoContent) + } + SubscriptionGroupWriteResult.InvalidName -> respond( + HttpStatusCode.BadRequest, + ErrorResponse("Group name must contain 1 to 100 characters", "subscription_group_invalid_name"), + ) + SubscriptionGroupWriteResult.DuplicateName -> respond( + HttpStatusCode.Conflict, + ErrorResponse("A subscription group with this name already exists", "subscription_group_name_conflict"), + ) + SubscriptionGroupWriteResult.NotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMembership(result: SubscriptionGroupMembershipResult) { + when (result) { + SubscriptionGroupMembershipResult.Success -> respond(HttpStatusCode.NoContent) + SubscriptionGroupMembershipResult.GroupNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + SubscriptionGroupMembershipResult.SubscriptionNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription not found", "subscription_not_found"), + ) + SubscriptionGroupMembershipResult.MembershipNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group membership not found", "subscription_group_membership_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMissingGroupId() = + respond(HttpStatusCode.BadRequest, ErrorResponse("Missing groupId")) From 46126970c810d88ba2d8ad1665d26c4e94ede17c Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:34:45 -0700 Subject: [PATCH 40/65] feat: filter subscription lists by group Parse group and ungrouped selectors for subscription reads, reject invalid or foreign group IDs, and separate subscription creation input from server-generated timestamps. --- .../models/SubscriptionCreateRequest.kt | 10 ++++++ .../routes/SubscriptionSelectionParameter.kt | 29 +++++++++++++++++ .../server/routes/SubscriptionsRoutes.kt | 32 ++++++++++++++++--- 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt new file mode 100644 index 00000000..0e530fdf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionCreateRequest( + val channelUrl: String, + val name: String, + val avatarUrl: String, +) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt new file mode 100644 index 00000000..a847aab0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SubscriptionSelection +import io.ktor.server.application.ApplicationCall + +internal sealed interface SubscriptionSelectionParseResult { + data class Valid(val selection: SubscriptionSelection) : SubscriptionSelectionParseResult + data object Invalid : SubscriptionSelectionParseResult +} + +internal fun ApplicationCall.parseSubscriptionSelection(): SubscriptionSelectionParseResult { + val rawGroupId = request.queryParameters["groupId"] + val groupId = rawGroupId?.takeIf(String::isNotBlank) + if (rawGroupId != null && groupId == null) return SubscriptionSelectionParseResult.Invalid + val rawUngrouped = request.queryParameters["ungrouped"] + val ungrouped = when (rawUngrouped) { + null -> false + "true" -> true + "false" -> false + else -> return SubscriptionSelectionParseResult.Invalid + } + if (groupId != null && ungrouped) return SubscriptionSelectionParseResult.Invalid + val selection = when { + groupId != null -> SubscriptionSelection.Group(groupId) + ungrouped -> SubscriptionSelection.Ungrouped + else -> SubscriptionSelection.All + } + return SubscriptionSelectionParseResult.Valid(selection) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 3d7b40b2..becd5d9b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -1,11 +1,14 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionCreateRequest import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.AuthService import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -18,16 +21,37 @@ import io.ktor.server.routing.post import java.net.URLDecoder import java.nio.charset.StandardCharsets -fun Route.subscriptionsRoutes(subscriptionsService: SubscriptionsService, authService: AuthService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup) { +fun Route.subscriptionsRoutes( + subscriptionsService: SubscriptionsService, + authService: AuthService, + warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), +) { get("/subscriptions") { - call.withJwtAuth(authService) { userId -> call.respond(subscriptionsService.getAll(userId)) } + call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + if (selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } + call.respond(subscriptionsService.getAll(userId, selection)) + } } post("/subscriptions") { call.withJwtAuth(authService) { userId -> - val item = runCatching { call.receive() }.getOrElse { + val request = runCatching { call.receive() }.getOrElse { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } - val subscription = subscriptionsService.add(userId, item) + val subscription = subscriptionsService.add( + userId, + SubscriptionItem(request.channelUrl, request.name, request.avatarUrl), + ) warmupService.invalidateAndWarm(userId) call.respond(HttpStatusCode.Created, subscription) } From 8665282d3d2bc500b987d89609543d71058d5439 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:34:53 -0700 Subject: [PATCH 41/65] feat: keep group feed pagination stable Project group and ungrouped feeds from the shared global snapshot while retaining each cursor's account-scoped membership selection in cache for the full pagination session. --- .../server/routes/SubscriptionFeedRoutes.kt | 17 ++++++- .../typetype/server/routes/UserDataRoutes.kt | 15 +++++- .../services/SubscriptionFeedBuilder.kt | 24 +++++++-- .../services/SubscriptionFeedCacheKeys.kt | 2 + .../SubscriptionFeedSelectionStore.kt | 50 +++++++++++++++++++ .../services/SubscriptionFeedService.kt | 19 ++++++- .../services/SubscriptionFeedSnapshot.kt | 50 ++++++++++++++++--- 7 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index e382da1c..2c3aabc8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -6,6 +6,8 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionFeedVisibility +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SettingsService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -19,12 +21,24 @@ fun Route.subscriptionFeedRoutes( feedService: SubscriptionFeedService, authService: AuthService, settingsService: SettingsService? = null, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), ) { get("/subscriptions/feed") { call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + val cursor = call.request.queryParameters["cursor"] + if (cursor == null && selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 - val cursor = call.request.queryParameters["cursor"] val visibility = settingsService?.subscriptionFeedVisibility(userId) ?: SubscriptionFeedVisibility() call.response.headers.append(HttpHeaders.CacheControl, "no-store") when ( @@ -35,6 +49,7 @@ fun Route.subscriptionFeedRoutes( cursor, visibility.hideLiveStreams, visibility.hideMembersOnlyContent, + selection, ) ) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 74baab91..1ff721c8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -17,8 +17,19 @@ internal fun Route.userDataRoutes( restoreService: PipePipeBackupImporterService, ) { historyRoutes(svc.historyService, authService, svc.settingsService) - subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) - subscriptionFeedRoutes(svc.subscriptionFeedService, authService, svc.settingsService) + subscriptionGroupsRoutes(svc.subscriptionGroupsService, authService) + subscriptionsRoutes( + svc.subscriptionsService, + authService, + svc.homeRecommendationWarmupService, + svc.subscriptionGroupsService, + ) + subscriptionFeedRoutes( + svc.subscriptionFeedService, + authService, + svc.settingsService, + svc.subscriptionGroupsService, + ) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index bae327dc..91621e5d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -22,13 +22,28 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } catch (error: CancellationException) { throw error } catch (_: Throwable) { - SubscriptionSourceResult(emptyList(), successfulSources = 0, failedSources = 1) + SubscriptionSourceResult( + channelUrl = subscription.channelUrl, + videos = emptyList(), + successfulSources = 0, + failedSources = 1, + ) } } }.map { it.await() } - val videos = outcomes.flatMap { it.videos }.deduplicated() + val videosByKey = linkedMapOf() + val sourceChannelUrls = linkedMapOf>() + outcomes.forEach { outcome -> + outcome.videos.forEach { video -> + val key = video.subscriptionFeedKey() + val current = videosByKey[key] + if (current == null || video.isLive && !current.isLive) videosByKey[key] = video + sourceChannelUrls.getOrPut(key, ::linkedSetOf).add(outcome.channelUrl) + } + } SubscriptionFeedBuildResult( - videos = videos, + videos = videosByKey.values.toList(), + sourceChannelUrls = sourceChannelUrls.mapValues { it.value.toList() }, successfulSources = outcomes.sumOf { it.successfulSources }, failedSources = outcomes.sumOf { it.failedSources }, ) @@ -46,6 +61,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } val results = listOfNotNull(channelResult, liveResult) SubscriptionSourceResult( + channelUrl = channelUrl, videos = mergeVideos(videos), successfulSources = results.count { it.success }, failedSources = results.count { !it.success }, @@ -91,6 +107,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private data class SourceFetchResult(val videos: List, val success: Boolean) private data class SubscriptionSourceResult( + val channelUrl: String, val videos: List, val successfulSources: Int, val failedSources: Int, @@ -105,6 +122,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic internal data class SubscriptionFeedBuildResult( val videos: List, + val sourceChannelUrls: Map>, val successfulSources: Int, val failedSources: Int, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt index d305a5da..af5f1eb0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt @@ -9,6 +9,8 @@ object SubscriptionFeedCacheKeys { fun invalidation(userId: String): String = "feed:invalidation:${hash(userId)}" + fun selection(userId: String, token: String): String = "feed:selection:${hash(userId)}:$token" + fun shorts(userId: String): String = "feed:shorts:${hash(userId)}" private fun hash(userId: String): String = MessageDigest diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt new file mode 100644 index 00000000..27d42252 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt @@ -0,0 +1,50 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import kotlinx.serialization.Serializable +import java.util.UUID + +internal class SubscriptionFeedSelectionStore( + private val cache: CacheService, + private val subscriptions: SubscriptionsService, +) { + suspend fun resolve( + userId: String, + selection: SubscriptionSelection, + token: String?, + ): SubscriptionFeedSelectionSnapshot? { + if (selection == SubscriptionSelection.All) return SubscriptionFeedSelectionSnapshot(null, null) + if (token == null) { + val channelUrls = subscriptions.getChannelUrls(userId, selection) + val nextToken = UUID.randomUUID().toString() + cache.set( + SubscriptionFeedCacheKeys.selection(userId, nextToken), + CacheJson.encodeToString( + StoredSubscriptionFeedSelection.serializer(), + StoredSubscriptionFeedSelection(selection.cursorKey, channelUrls.toList()), + ), + SubscriptionFeedSnapshotStore.RETENTION_SECONDS, + ) + return SubscriptionFeedSelectionSnapshot(nextToken, channelUrls) + } + val raw = runCatching { cache.get(SubscriptionFeedCacheKeys.selection(userId, token)) }.getOrNull() + ?: return null + val stored = runCatching { + CacheJson.decodeFromString(StoredSubscriptionFeedSelection.serializer(), raw) + }.getOrNull() ?: return null + if (stored.filterKey != selection.cursorKey) return null + return SubscriptionFeedSelectionSnapshot(token, stored.channelUrls.toSet()) + } +} + +@Serializable +private data class StoredSubscriptionFeedSelection( + val filterKey: String, + val channelUrls: List, +) + +internal data class SubscriptionFeedSelectionSnapshot( + val token: String?, + val channelUrls: Set?, +) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index de1da91c..38e75d8d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -26,6 +26,7 @@ class SubscriptionFeedService( private val refreshScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), ) { private val store = SubscriptionFeedSnapshotStore(cache, clock) + private val selections = SubscriptionFeedSelectionStore(cache, subscriptionsService) private val builder = SubscriptionFeedBuilder(channelService) private val orderer = SubscriptionFeedOrderer() private val refreshJobs = ConcurrentHashMap() @@ -38,6 +39,7 @@ class SubscriptionFeedService( cursor: String?, hideLiveStreams: Boolean = false, hideMembersOnlyContent: Boolean = false, + selection: SubscriptionSelection = SubscriptionSelection.All, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -57,6 +59,9 @@ class SubscriptionFeedService( if (cursorState != null && cursorState.hideMembersOnlyContent != hideMembersOnlyContent) { return SubscriptionFeedPageResult.InvalidCursor } + if (cursorState != null && cursorState.filterKey != selection.cursorKey) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current @@ -64,8 +69,19 @@ class SubscriptionFeedService( ?: return SubscriptionFeedPageResult.StaleGeneration } val offset = cursorState?.offset ?: page * limit + val selected = selections.resolve(userId, selection, cursorState?.selectionToken) + ?: return SubscriptionFeedPageResult.StaleGeneration return SubscriptionFeedPageResult.Ready( - snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams, hideMembersOnlyContent), + snapshot.page( + offset, + limit, + isRefreshing(userId), + hideLiveStreams, + hideMembersOnlyContent, + selection, + selected.channelUrls, + selected.token, + ), ) } @@ -152,6 +168,7 @@ class SubscriptionFeedService( stale = false, videos = ordering.videos, livePromotedAt = ordering.livePromotedAt, + sourceChannelUrls = result.sourceChannelUrls, ) runCatching { store.publish(userId, snapshot) }.onFailure { logger.warn("subscription_feed event=publish_failed user={} error={}", userKey(userId), it.message) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index c752034a..5d8232a1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -13,6 +13,7 @@ internal data class SubscriptionFeedSnapshot( val stale: Boolean, val videos: List, val livePromotedAt: Map = emptyMap(), + val sourceChannelUrls: Map> = emptyMap(), ) @Serializable @@ -22,6 +23,8 @@ private data class SubscriptionFeedCursor( val limit: Int, val hideLiveStreams: Boolean = false, val hideMembersOnlyContent: Boolean = false, + val filterKey: String = SubscriptionSelection.All.cursorKey, + val selectionToken: String? = null, ) internal object SubscriptionFeedCursorCodec { @@ -31,10 +34,14 @@ internal object SubscriptionFeedCursorCodec { limit: Int, hideLiveStreams: Boolean, hideMembersOnlyContent: Boolean, + filterKey: String, + selectionToken: String?, ): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams, hideMembersOnlyContent), + SubscriptionFeedCursor( + generation, offset, limit, hideLiveStreams, hideMembersOnlyContent, filterKey, selectionToken, + ), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -42,7 +49,10 @@ internal object SubscriptionFeedCursorCodec { fun decode(value: String): SubscriptionFeedCursorState? = runCatching { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) - cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } + cursor.takeIf { + it.generation > 0L && it.offset >= 0 && it.limit in 1..100 && + ((it.filterKey == SubscriptionSelection.All.cursorKey) == (it.selectionToken == null)) + } ?.let { SubscriptionFeedCursorState( it.generation, @@ -50,6 +60,8 @@ internal object SubscriptionFeedCursorCodec { it.limit, it.hideLiveStreams, it.hideMembersOnlyContent, + it.filterKey, + it.selectionToken, ) } }.getOrNull() @@ -61,6 +73,8 @@ internal data class SubscriptionFeedCursorState( val limit: Int, val hideLiveStreams: Boolean, val hideMembersOnlyContent: Boolean, + val filterKey: String, + val selectionToken: String?, ) internal fun SubscriptionFeedSnapshot.page( @@ -69,26 +83,31 @@ internal fun SubscriptionFeedSnapshot.page( refreshing: Boolean, hideLiveStreams: Boolean = false, hideMembersOnlyContent: Boolean = false, + selection: SubscriptionSelection = SubscriptionSelection.All, + selectedChannelUrls: Set? = null, + selectionToken: String? = null, ): SubscriptionFeedResponse { - val visibleVideos = videos.filterNot { video -> + val projectedVideos = projectedVideos(selection, selectedChannelUrls).filterNot { video -> (hideLiveStreams && video.isLiveOrUpcomingAt(generatedAt)) || (hideMembersOnlyContent && video.requiresMembership) } - val from = offset.coerceAtMost(visibleVideos.size) - val to = minOf(from + limit, visibleVideos.size) - val nextpage = if (to < visibleVideos.size) { + val from = offset.coerceAtMost(projectedVideos.size) + val to = minOf(from + limit, projectedVideos.size) + val nextpage = if (to < projectedVideos.size) { SubscriptionFeedCursorCodec.encode( generation, to, limit, hideLiveStreams, hideMembersOnlyContent, + selection.cursorKey, + selectionToken, ) } else { null } return SubscriptionFeedResponse( - videos = visibleVideos.subList(from, to), + videos = projectedVideos.subList(from, to), nextpage = nextpage, generation = generation, generatedAt = generatedAt, @@ -96,6 +115,23 @@ internal fun SubscriptionFeedSnapshot.page( ) } +private fun SubscriptionFeedSnapshot.projectedVideos( + selection: SubscriptionSelection, + selectedChannelUrls: Set?, +): List { + if (selection == SubscriptionSelection.All) return videos + val allowed = selectedChannelUrls.orEmpty() + if (allowed.isEmpty()) return emptyList() + return videos.filter { video -> + val sources = sourceChannelUrls[video.subscriptionFeedKey()] + if (sources != null) { + sources.any { ChannelUrlCanonicalizer.canonicalize(it) in allowed } + } else { + ChannelUrlCanonicalizer.canonicalize(video.uploaderUrl) in allowed + } + } +} + internal sealed interface SubscriptionFeedPageResult { data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult From 860d6869709165b98bf4a12a616fd68e85e7c763 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:35:00 -0700 Subject: [PATCH 42/65] feat: preserve subscription groups in backups Export named groups with their channel memberships and restore them transactionally with subscriptions. Validate names and membership references before replacing account-owned group data. --- openapi/components/user-backup.yaml | 12 ++++ .../models/SubscriptionGroupBackupItem.kt | 11 ++++ .../server/models/TypeTypeBackupItem.kt | 1 + .../SubscriptionGroupBackupRepository.kt | 64 +++++++++++++++++++ .../services/TypeTypeBackupRestoreWriter.kt | 5 ++ .../server/services/TypeTypeBackupService.kt | 37 +++++++++++ .../server/TypeTypeBackupServiceTest.kt | 19 ++++++ 7 files changed, 149 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt diff --git a/openapi/components/user-backup.yaml b/openapi/components/user-backup.yaml index 36f5a1d4..76428dd8 100644 --- a/openapi/components/user-backup.yaml +++ b/openapi/components/user-backup.yaml @@ -23,6 +23,17 @@ TypeTypeContentFiltersBackup: allowedPlaylists: type: array items: { $ref: ./access-control.yaml#/AllowedPlaylistItem } +SubscriptionGroupBackupItem: + type: object + required: [name, channelUrls, createdAt, updatedAt] + properties: + name: { type: string, minLength: 1, maxLength: 100 } + channelUrls: + type: array + uniqueItems: true + items: { type: string, minLength: 1 } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } TypeTypeBackupItem: type: object required: [format, version, exportedAt, categories] @@ -47,6 +58,7 @@ TypeTypeBackupItem: - settings - contentFilters subscriptions: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + subscriptionGroups: { type: array, nullable: true, items: { $ref: '#/SubscriptionGroupBackupItem' } } history: { type: array, nullable: true, items: { type: object, additionalProperties: true } } playlists: { type: array, nullable: true, items: { type: object, additionalProperties: true } } watchLater: { type: array, nullable: true, items: { type: object, additionalProperties: true } } diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt new file mode 100644 index 00000000..efae0cab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupBackupItem( + val name: String, + val channelUrls: List, + val createdAt: Long, + val updatedAt: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt index 4c9300b9..82cb9f40 100644 --- a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt @@ -9,6 +9,7 @@ data class TypeTypeBackupItem( val exportedAt: Long, val categories: List, val subscriptions: List? = null, + val subscriptionGroups: List? = null, val history: List? = null, val playlists: List? = null, val watchLater: List? = null, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt new file mode 100644 index 00000000..3cc685aa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt @@ -0,0 +1,64 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.models.SubscriptionGroupBackupItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.util.Locale +import java.util.UUID + +internal object SubscriptionGroupBackupRepository { + suspend fun export(userId: String): List = DatabaseFactory.query { + val channelsByGroup = SubscriptionGroupMembershipsTable.selectAll() + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.ASC) + .groupBy( + keySelector = { it[SubscriptionGroupMembershipsTable.groupId] }, + valueTransform = { it[SubscriptionGroupMembershipsTable.channelUrl] }, + ) + SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.ASC) + .map { row -> + SubscriptionGroupBackupItem( + name = row[SubscriptionGroupsTable.name], + channelUrls = channelsByGroup[row[SubscriptionGroupsTable.id]].orEmpty(), + createdAt = row[SubscriptionGroupsTable.createdAt], + updatedAt = row[SubscriptionGroupsTable.updatedAt], + ) + } + } + + fun restore(userId: String, items: List): Pair { + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } + val groups = items.map { it to UUID.randomUUID().toString() } + if (groups.isNotEmpty()) { + SubscriptionGroupsTable.batchInsert(groups, shouldReturnGeneratedValues = false) { (item, id) -> + this[SubscriptionGroupsTable.id] = id + this[SubscriptionGroupsTable.userId] = userId + this[SubscriptionGroupsTable.name] = item.name + this[SubscriptionGroupsTable.normalizedName] = item.name.lowercase(Locale.ROOT) + this[SubscriptionGroupsTable.createdAt] = item.createdAt + this[SubscriptionGroupsTable.updatedAt] = item.updatedAt + } + } + val memberships = groups.flatMap { (item, groupId) -> + item.channelUrls.map { channelUrl -> groupId to ChannelUrlCanonicalizer.canonicalize(channelUrl) } + } + if (memberships.isNotEmpty()) { + SubscriptionGroupMembershipsTable.batchInsert(memberships, shouldReturnGeneratedValues = false) { (groupId, channelUrl) -> + this[SubscriptionGroupMembershipsTable.groupId] = groupId + this[SubscriptionGroupMembershipsTable.userId] = userId + this[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + this[SubscriptionGroupMembershipsTable.addedAt] = System.currentTimeMillis() + } + } + return groups.size to memberships.size + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt index ad10644a..d951dafd 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -16,6 +16,11 @@ internal object TypeTypeBackupRestoreWriter { userId, requireNotNull(backup.subscriptions), ) + backup.subscriptionGroups?.let { groups -> + val counts = SubscriptionGroupBackupRepository.restore(userId, groups) + restored["subscriptionGroups"] = counts.first + restored["subscriptionGroupMemberships"] = counts.second + } } if (TypeTypeBackupCategory.HISTORY in categories) { restored["history"] = TypeTypeBackupCoreRestore.history( diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt index 1c98ddfb..97466980 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION import dev.typetype.server.models.TypeTypeBackupItem import dev.typetype.server.models.TypeTypeContentFiltersBackup import dev.typetype.server.models.TypeTypeRestoreSummary +import java.util.Locale class TypeTypeBackupService( private val subscriptions: SubscriptionsService, @@ -34,6 +35,11 @@ class TypeTypeBackupService( exportedAt = System.currentTimeMillis(), categories = categories.map(TypeTypeBackupCategory::wireName).sorted(), subscriptions = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) subscriptions.getAll(userId) else null, + subscriptionGroups = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) { + SubscriptionGroupBackupRepository.export(userId) + } else { + null + }, history = if (includes(TypeTypeBackupCategory.HISTORY)) history.getAll(userId) else null, playlists = fullPlaylists, watchLater = if (includes(TypeTypeBackupCategory.WATCH_LATER)) watchLater.getAll(userId) else null, @@ -53,6 +59,7 @@ class TypeTypeBackupService( val categories = TypeTypeBackupCategory.parse(backup.categories.joinToString(",")) ?: throw IllegalArgumentException("Invalid backup categories") validateSections(backup, categories) + validateSubscriptionGroups(backup, categories) validateContentFilters(backup, categories) return TypeTypeBackupRestoreWriter.restore(userId, backup, categories) } @@ -87,6 +94,36 @@ private fun validateSections( require(missing.isEmpty()) { "Backup is missing selected data" } } +private fun validateSubscriptionGroups( + backup: TypeTypeBackupItem, + categories: Set, +) { + val groups = backup.subscriptionGroups ?: return + require(TypeTypeBackupCategory.SUBSCRIPTIONS in categories) { + "Subscription groups require the subscriptions category" + } + val normalizedNames = groups.map { group -> + require(group.name == group.name.trim() && group.name.length in 1..SubscriptionGroupsService.MAX_GROUP_NAME_LENGTH) { + "Subscription group names must contain 1 to 100 characters" + } + group.name.lowercase(Locale.ROOT) + } + require(normalizedNames.distinct().size == normalizedNames.size) { + "Backup contains duplicate subscription group names" + } + val subscriptions = requireNotNull(backup.subscriptions) + .mapTo(mutableSetOf()) { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + groups.forEach { group -> + val channels = group.channelUrls.map(ChannelUrlCanonicalizer::canonicalize) + require(channels.distinct().size == channels.size) { + "Backup contains duplicate subscription group memberships" + } + require(channels.all { it in subscriptions }) { + "Subscription group membership references an unknown subscription" + } + } +} + private fun validateContentFilters( backup: TypeTypeBackupItem, categories: Set, diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt index 8cce3567..7d9a2219 100644 --- a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt @@ -23,6 +23,9 @@ import dev.typetype.server.services.ProgressService import dev.typetype.server.services.SavedPlaylistService import dev.typetype.server.services.SearchHistoryService import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.TypeTypeBackupCategory import dev.typetype.server.services.TypeTypeBackupService @@ -37,6 +40,7 @@ import org.junit.jupiter.api.Test class TypeTypeBackupServiceTest { private val subscriptions = SubscriptionsService() + private val subscriptionGroups = SubscriptionGroupsService() private val history = HistoryService() private val playlists = PlaylistService() private val watchLater = WatchLaterService() @@ -78,6 +82,13 @@ class TypeTypeBackupServiceTest { @Test fun `full backup restores every user data category`() = runTest { subscriptions.add(SOURCE, SubscriptionItem("https://youtube.com/channel/source", "Source", "avatar")) + val subscriptionGroup = ( + subscriptionGroups.create(SOURCE, "Favorites") as SubscriptionGroupWriteResult.Success + ).group + assertEquals( + SubscriptionGroupMembershipResult.Success, + subscriptionGroups.addSubscription(SOURCE, subscriptionGroup.id, "https://youtube.com/channel/source"), + ) history.addImported(SOURCE, videoHistory()) val playlist = playlists.create(SOURCE, PlaylistItem(name = "Saved videos")) playlists.addVideo(SOURCE, playlist.id, playlistVideo()) @@ -107,6 +118,14 @@ class TypeTypeBackupServiceTest { val result = service.restore(TARGET, backup) assertEquals(1, result.restored["subscriptions"]) + assertEquals(1, result.restored["subscriptionGroups"]) + assertEquals(1, result.restored["subscriptionGroupMemberships"]) + val restoredGroup = subscriptionGroups.getAll(TARGET).single() + assertEquals("Favorites", restoredGroup.name) + assertEquals( + listOf("https://youtube.com/channel/source"), + subscriptionGroups.getChannelUrls(TARGET, restoredGroup.id), + ) assertEquals(1, result.restored["history"]) assertEquals(1, result.restored["playlists"]) assertEquals(1, result.restored["playlistVideos"]) From f756bfab3dbf9e5eaeca2399fa4282ae8138c897 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:35:05 -0700 Subject: [PATCH 43/65] docs: document subscription group endpoints Describe group management and filtered list/feed operations, and make subscription creation use a request schema without the server-generated subscribedAt field. --- openapi.yaml | 8 + openapi/components/subscriptions.yaml | 34 ++++ openapi/paths/subscriptions.yaml | 157 +++++++++++++++++- .../server/SubscriptionsRoutesTest.kt | 16 ++ 4 files changed, 214 insertions(+), 1 deletion(-) diff --git a/openapi.yaml b/openapi.yaml index f551cc64..37be9bbe 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -43,6 +43,10 @@ paths: /playlist: { $ref: ./openapi/paths/playlists.yaml#/Playlist } /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } + /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } + /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } + /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } + /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } @@ -146,6 +150,10 @@ components: $ref: ./openapi/components/media.yaml#/PublicPlaylistItem SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } + SubscriptionItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionItem } + SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } + SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } + SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index b0e7ad59..5894cc5f 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -1,3 +1,37 @@ +SubscriptionItem: + type: object + required: [channelUrl, name, avatarUrl, subscribedAt] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } + subscribedAt: { type: integer, format: int64 } +SubscriptionCreateRequest: + type: object + required: [channelUrl, name, avatarUrl] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } +SubscriptionGroupItem: + type: object + required: [id, name, channelCount, createdAt, updatedAt] + properties: + id: { type: string, format: uuid } + name: { type: string, minLength: 1, maxLength: 100 } + channelCount: { type: integer, minimum: 0 } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } +SubscriptionGroupRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 100 } +SubscriptionGroupMembershipRequest: + type: object + required: [channelUrl] + properties: + channelUrl: { type: string, minLength: 1 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index f0dcc65c..ab9bee92 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -1,8 +1,161 @@ +Subscriptions: + get: + tags: [user-data] + summary: List the current user's subscriptions + description: Omit both filters for the global list. Use groupId for one named group or ungrouped=true for subscriptions in no groups. + parameters: + - name: groupId + in: query + required: false + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + schema: { type: boolean, default: false } + responses: + '200': + description: The selected subscription projection. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Subscribe to a channel + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionCreateRequest } + responses: + '201': + description: Subscription created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Unsubscribe from a channel + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Subscription deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroups: + get: + tags: [user-data] + summary: List the current user's subscription groups + responses: + '200': + description: Account-scoped named groups. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '401': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Create a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '201': + description: Group created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroup: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Rename a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '204': { description: Group renamed. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Delete a subscription group + responses: + '204': { description: Group and its memberships deleted. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroupChannels: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Add a subscribed channel to a group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + responses: + '204': { description: Membership exists. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Remove a subscribed channel from a group + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Membership deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } SubscriptionFeed: get: tags: [user-data] summary: Read a stable page from the current user's subscription feed snapshot parameters: + - name: groupId + in: query + required: false + description: Restrict the snapshot projection to subscriptions in one account-owned group. + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + description: Restrict the snapshot projection to subscriptions in no groups. + schema: { type: boolean, default: false } - name: page in: query required: false @@ -16,7 +169,7 @@ SubscriptionFeed: - name: cursor in: query required: false - description: Opaque continuation returned in nextpage. + description: Opaque continuation returned in nextpage and bound to the selected membership snapshot. schema: { type: string } responses: '200': @@ -45,6 +198,8 @@ SubscriptionFeed: $ref: ../components/common.yaml#/JsonError '401': $ref: ../components/common.yaml#/JsonError + '404': + $ref: ../components/common.yaml#/JsonError '409': description: The cursor references a snapshot generation that is no longer retained. headers: diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt index d3075356..df2da9ac 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt @@ -20,10 +20,13 @@ import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Path class SubscriptionsRoutesTest { @@ -49,6 +52,19 @@ class SubscriptionsRoutesTest { private val itemBody = """{"channelUrl":"https://yt.com/channel/1","name":"Test","avatarUrl":""}""" + @Test + fun `subscription creation contract omits the server timestamp`() { + val components = Files.readString(Path.of("openapi/components/subscriptions.yaml")) + val requestSchema = components + .substringAfter("SubscriptionCreateRequest:") + .substringBefore("SubscriptionGroupItem:") + val paths = Files.readString(Path.of("openapi/paths/subscriptions.yaml")) + + assertTrue("required: [channelUrl, name, avatarUrl]" in requestSchema) + assertFalse("subscribedAt" in requestSchema) + assertTrue("#/SubscriptionCreateRequest" in paths) + } + @Test fun `GET subscriptions without token returns 401`() = withApp { assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions").status) From 8336b00362762dab731ba64327a01b3931078a01 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:35:11 -0700 Subject: [PATCH 44/65] test: cover subscription group persistence Verify account isolation, normalized unique names, many-to-many membership, ungrouped selection, and cleanup after subscription replacement or deletion. --- .../server/SubscriptionGroupsServiceTest.kt | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt new file mode 100644 index 00000000..d03bce44 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -0,0 +1,120 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionGroupMembershipCleaner +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupsServiceTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `group names are normalized unique and account scoped`() = runTest { + val group = groups.create("user-a", " Work ").createdGroup() + + assertEquals("Work", group.name) + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.create("user-a", "work")) + assertTrue(groups.create("user-b", "work") is SubscriptionGroupWriteResult.Success) + assertFalse(groups.exists("user-b", group.id)) + assertEquals( + SubscriptionGroupWriteResult.NotFound, + groups.rename("user-b", group.id, "Other"), + ) + groups.create("user-a", "Other") + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.rename("user-a", group.id, "OTHER")) + } + + @Test + fun `a subscription can belong to multiple groups while ungrouped stays distinct`() = runTest { + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + subscriptions.add("user", subscription("three")) + val first = groups.create("user", "First").createdGroup() + val second = groups.create("user", "Second").createdGroup() + + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", first.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("two"))) + assertEquals(1, groups.getAll("user").first { it.id == first.id }.channelCount) + + assertEquals( + listOf(channel("one")), + subscriptions.getAll("user", SubscriptionSelection.Group(first.id)).map { it.channelUrl }, + ) + assertEquals( + setOf(channel("one"), channel("two")), + subscriptions.getAll("user", SubscriptionSelection.Group(second.id)).map { it.channelUrl }.toSet(), + ) + assertEquals( + listOf(channel("three")), + subscriptions.getAll("user", SubscriptionSelection.Ungrouped).map { it.channelUrl }, + ) + } + + @Test + fun `membership requires both the users group and subscription`() = runTest { + val group = groups.create("user-a", "A").createdGroup() + subscriptions.add("user-b", subscription("shared")) + + assertEquals( + SubscriptionGroupMembershipResult.SubscriptionNotFound, + groups.addSubscription("user-a", group.id, channel("shared")), + ) + assertEquals( + SubscriptionGroupMembershipResult.GroupNotFound, + groups.addSubscription("user-b", group.id, channel("shared")), + ) + } + + @Test + fun `deleting a subscription removes its memberships`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + groups.addSubscription("user", group.id, channel("one")) + + assertTrue(subscriptions.delete("user", channel("one"))) + + assertEquals(emptyList(), groups.getChannelUrls("user", group.id)) + } + + @Test + fun `replacement imports retain only memberships for subscriptions still present`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + groups.addSubscription("user", group.id, channel("one")) + groups.addSubscription("user", group.id, channel("two")) + + DatabaseFactory.query { SubscriptionGroupMembershipCleaner.retain("user", listOf(channel("one"))) } + + assertEquals(listOf(channel("one")), groups.getChannelUrls("user", group.id)) + } + + private fun SubscriptionGroupWriteResult.createdGroup() = + (this as SubscriptionGroupWriteResult.Success).group + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://yt.com/channel/$id" +} From 939158ab4a2e43aabec5dc266f10ec4b6d260026 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:35:16 -0700 Subject: [PATCH 45/65] test: cover subscription group routes Exercise authenticated group CRUD, membership updates, filter validation, and cross-account access through the HTTP routing surface. --- .../server/SubscriptionGroupsRoutesTest.kt | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt new file mode 100644 index 00000000..70dd04b3 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -0,0 +1,161 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionGroupItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionGroupsRoutes +import dev.typetype.server.routes.subscriptionsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.request.post +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupsRoutesTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + subscriptionGroupsRoutes(groups, auth) + subscriptionsRoutes(subscriptions, auth, groupsService = groups) + } + } + block() + } + + @Test + fun `group routes require authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/groups").status) + } + + @Test + fun `groups can be created listed renamed and deleted`() = withApp { + val create = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + } + assertEquals(HttpStatusCode.Created, create.status) + val group = Json.decodeFromString(create.bodyAsText()) + + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Work\"")) + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}") { + authorizeJson() + setBody("""{"name":"Research"}""") + }.status) + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Research\"")) + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}") { authorize() }.status) + assertEquals("[]", authorizedGet("/subscriptions/groups").bodyAsText()) + } + + @Test + fun `blank and duplicate group names are rejected`() = withApp { + assertEquals(HttpStatusCode.BadRequest, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":" "}""") + }.status) + assertEquals(HttpStatusCode.Created, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + }.status) + assertEquals(HttpStatusCode.Conflict, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"work"}""") + }.status) + } + + @Test + fun `membership drives grouped and ungrouped subscription projections`() = withApp { + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("one"), "One", "")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("two"), "Two", "")) + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"${channel("one")}"}""") + }.status) + + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) } + assertTrue(grouped.bodyAsText().contains(channel("one"))) + assertTrue(!grouped.bodyAsText().contains(channel("two"))) + val ungrouped = authorizedGet("/subscriptions") { parameter("ungrouped", true) } + assertTrue(!ungrouped.bodyAsText().contains(channel("one"))) + assertTrue(ungrouped.bodyAsText().contains(channel("two"))) + + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}/channels") { + authorize() + parameter("url", channel("one")) + }.status) + assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) + } + + @Test + fun `invalid or inaccessible filters fail explicitly`() = withApp { + assertEquals(HttpStatusCode.BadRequest, authorizedGet("/subscriptions") { + parameter("groupId", "group") + parameter("ungrouped", true) + }.status) + assertEquals(HttpStatusCode.NotFound, authorizedGet("/subscriptions") { + parameter("groupId", "missing") + }.status) + } + + private suspend fun ApplicationTestBuilder.createGroup(name: String): SubscriptionGroupItem { + val response = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"$name"}""") + } + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.authorizedGet( + path: String, + configure: io.ktor.client.request.HttpRequestBuilder.() -> Unit = {}, + ) = client.get(path) { + authorize() + configure() + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorizeJson() { + authorize() + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } + + private fun channel(id: String) = "https://yt.com/channel/$id" +} From af4cd5c1e607ceef051f66f80db59c3562ad8a4c Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 15:35:21 -0700 Subject: [PATCH 46/65] test: cover stable subscription group feeds Verify shared-snapshot projection, source-channel attribution, filter-bound cursors, and unchanged membership snapshots across paginated group reads. --- .../server/SubscriptionGroupFeedRoutesTest.kt | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt new file mode 100644 index 00000000..c2becbaa --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt @@ -0,0 +1,168 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionFeedRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupFeedRoutesTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + private lateinit var feed: SubscriptionFeedService + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + feed = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionFeedRoutes(feed, auth, groupsService = groups) } + } + block() + } + + @Test + fun `group and ungrouped feeds project one shared global snapshot`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + assertEquals( + SubscriptionGroupMembershipResult.Success, + groups.addSubscription(TEST_USER_ID, group.id, channel("one")), + ) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("${channel("one")}/video"), requestReadyFeed(groupId = group.id).videos.map { it.url }) + assertEquals(listOf("${channel("two")}/video"), requestReadyFeed(ungrouped = true).videos.map { it.url }) + assertEquals(2, requestReadyFeed().videos.size) + } + + @Test + fun `cursor keeps its original group membership across pages`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(channel("one"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(3_000L, channel = "one", url = "video-one"), + ) + coEvery { channelService.getChannel(channel("two"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(2_000L, channel = "two", url = "video-two"), + ) + coEvery { channelService.getChannel(channel("three"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(1_000L, channel = "three", url = "video-three"), + ) + feed = SubscriptionFeedService(subscriptions, channelService, FakeCacheService()) + listOf("one", "two", "three").forEach { subscriptions.add(TEST_USER_ID, subscription(it)) } + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + groups.addSubscription(TEST_USER_ID, group.id, channel("two")) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1, groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + val firstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertEquals(listOf("video-one"), firstPage.videos.map { it.url }) + + groups.removeSubscription(TEST_USER_ID, group.id, channel("two")) + groups.addSubscription(TEST_USER_ID, group.id, channel("three")) + val secondPage = requestFeed(limit = 1, cursor = requireNotNull(firstPage.nextpage), groupId = group.id) + + assertEquals(HttpStatusCode.OK, secondPage.status) + assertEquals(listOf("video-two"), Json.decodeFromString(secondPage.bodyAsText()).videos.map { it.url }) + } + + @Test + fun `cursor cannot be reused with another subscription filter`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feed.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(requestReadyFeed(limit = 1).nextpage) + + val response = requestFeed(limit = 1, cursor = cursor, groupId = group.id) + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("subscription_feed_invalid_cursor")) + } + + @Test + fun `group feed follows the fetched subscription source when uploader url differs`() = withApp { + val sourceUrl = channel("one") + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, sourceUrl) + val channelService = mockk() + coEvery { channelService.getChannel(sourceUrl, null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(1_000L, channel = "different-canonical-uploader"), + ) + feed = SubscriptionFeedService(subscriptions, channelService, FakeCacheService()) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + } + + private suspend fun ApplicationTestBuilder.requestReadyFeed( + limit: Int = 30, + groupId: String? = null, + ungrouped: Boolean = false, + ): SubscriptionFeedResponse { + val response = requestFeed(limit = limit, groupId = groupId, ungrouped = ungrouped) + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.requestFeed( + limit: Int = 30, + cursor: String? = null, + groupId: String? = null, + ungrouped: Boolean = false, + ): HttpResponse = client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + groupId?.let { parameter("groupId", it) } + if (ungrouped) parameter("ungrouped", true) + } + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://example.com/channel/$id" +} From abd1ae9aec6e19120ad4bd5fb66e3c1c5c46e183 Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 16:09:13 -0700 Subject: [PATCH 47/65] fix: bound feed membership snapshot storage Filtered pagination needs a stable membership view without creating an unbounded cache entry for every initial request. Allocate a fixed set of account-scoped slots atomically, reuse content-derived tokens, and reject a new distinct session at capacity without evicting any issued cursor. Strengthen observable compatibility coverage for legacy backups and server timestamps. Constraint: Every issued cursor must retain its membership snapshot for the full cache TTL Rejected: Evict the oldest snapshot after eight sessions | invalidates a still-live cursor Rejected: Store all snapshots in one read-modify-write value | loses concurrent writes across server instances Rejected: Put channel URLs directly in the cursor | produces oversized client-controlled cursor payloads Confidence: high Scope-risk: moderate Reversibility: clean Directive: Never overwrite an occupied selection slot; reject new sessions before weakening issued cursors Tested: Focused feed, concurrent selection-store, subscription route, backup, and OpenAPI tests on JDK 25 Not-tested: Live Dragonfly slot saturation before the final runtime gate --- openapi/paths/subscriptions.yaml | 9 ++ .../dev/typetype/server/cache/CacheService.kt | 4 + .../typetype/server/cache/DragonflyService.kt | 19 +++++ .../server/routes/SubscriptionFeedRoutes.kt | 8 ++ .../services/SubscriptionFeedCacheKeys.kt | 2 +- .../SubscriptionFeedSelectionStore.kt | 84 +++++++++++++++---- .../services/SubscriptionFeedService.kt | 24 +++--- .../services/SubscriptionFeedSnapshot.kt | 6 +- .../dev/typetype/server/FakeCacheService.kt | 14 ++++ .../SubscriptionFeedSelectionStoreTest.kt | 73 ++++++++++++++++ .../server/SubscriptionGroupFeedRoutesTest.kt | 26 +++++- .../server/SubscriptionsRoutesTest.kt | 40 ++++----- .../server/TypeTypeBackupServiceTest.kt | 25 ++++++ 13 files changed, 285 insertions(+), 49 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index ab9bee92..9f9b4863 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -209,3 +209,12 @@ SubscriptionFeed: application/json: schema: $ref: ../components/common.yaml#/ErrorResponse + '429': + description: The account already has the maximum number of active filtered cursor sessions. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt b/src/main/kotlin/dev/typetype/server/cache/CacheService.kt index cd67eb79..40f16770 100644 --- a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/CacheService.kt @@ -3,5 +3,9 @@ package dev.typetype.server.cache interface CacheService { suspend fun get(key: String): String? suspend fun set(key: String, value: String, ttlSeconds: Long) + suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + throw UnsupportedOperationException("Atomic set-if-absent is not supported") + suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean = + throw UnsupportedOperationException("Atomic compare-and-expire is not supported") suspend fun delete(key: String) } diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt index 32aa51b2..d8517ccd 100644 --- a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt @@ -1,6 +1,8 @@ package dev.typetype.server.cache import io.lettuce.core.RedisClient +import io.lettuce.core.ScriptOutputType +import io.lettuce.core.SetArgs import io.lettuce.core.api.StatefulRedisConnection import io.lettuce.core.api.async.RedisAsyncCommands import kotlinx.coroutines.future.await @@ -17,8 +19,25 @@ class DragonflyService(url: String) : CacheService { override suspend fun set(key: String, value: String, ttlSeconds: Long): Unit = async.setex(key, ttlSeconds, value).await().let {} + override suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + async.set(key, value, SetArgs.Builder.nx().ex(ttlSeconds)).await() == "OK" + + override suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean = + async.eval( + REFRESH_IF_VALUE_MATCHES, + ScriptOutputType.INTEGER, + arrayOf(key), + value, + ttlSeconds.toString(), + ).await() == 1L + override suspend fun delete(key: String): Unit = async.del(key).await().let {} suspend fun ping(): Boolean = async.ping().await() == "PONG" + + private companion object { + const val REFRESH_IF_VALUE_MATCHES = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('expire', KEYS[1], ARGV[2]) end return 0" + } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 2c3aabc8..abdfb136 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -2,6 +2,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.SubscriptionFeedPreparingResponse +import dev.typetype.server.preserveTooManyRequestsBody import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService @@ -68,6 +69,13 @@ fun Route.subscriptionFeedRoutes( HttpStatusCode.Conflict, ErrorResponse("Subscription feed generation is no longer available", "subscription_feed_stale_generation"), ) + SubscriptionFeedPageResult.CursorCapacityReached -> { + call.preserveTooManyRequestsBody() + call.respond( + HttpStatusCode.TooManyRequests, + ErrorResponse("Too many active subscription feed cursors", "subscription_feed_cursor_capacity"), + ) + } } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt index af5f1eb0..863bbba9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt @@ -9,7 +9,7 @@ object SubscriptionFeedCacheKeys { fun invalidation(userId: String): String = "feed:invalidation:${hash(userId)}" - fun selection(userId: String, token: String): String = "feed:selection:${hash(userId)}:$token" + fun selection(userId: String, slot: Int): String = "feed:selection:${hash(userId)}:$slot" fun shorts(userId: String): String = "feed:shorts:${hash(userId)}" diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt index 27d42252..b8bc1737 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheJson import dev.typetype.server.cache.CacheService import kotlinx.serialization.Serializable -import java.util.UUID +import java.security.MessageDigest internal class SubscriptionFeedSelectionStore( private val cache: CacheService, @@ -17,29 +17,83 @@ internal class SubscriptionFeedSelectionStore( if (selection == SubscriptionSelection.All) return SubscriptionFeedSelectionSnapshot(null, null) if (token == null) { val channelUrls = subscriptions.getChannelUrls(userId, selection) - val nextToken = UUID.randomUUID().toString() - cache.set( - SubscriptionFeedCacheKeys.selection(userId, nextToken), - CacheJson.encodeToString( - StoredSubscriptionFeedSelection.serializer(), - StoredSubscriptionFeedSelection(selection.cursorKey, channelUrls.toList()), - ), - SubscriptionFeedSnapshotStore.RETENTION_SECONDS, + return SubscriptionFeedSelectionSnapshot( + token = tokenFor(selection.cursorKey, channelUrls), + channelUrls = channelUrls, ) - return SubscriptionFeedSelectionSnapshot(nextToken, channelUrls) } - val raw = runCatching { cache.get(SubscriptionFeedCacheKeys.selection(userId, token)) }.getOrNull() + for (slot in 0 until MAX_SNAPSHOTS_PER_USER) { + val stored = read(userId, slot) ?: continue + if (stored.token != token) continue + if (stored.filterKey != selection.cursorKey) return null + return SubscriptionFeedSelectionSnapshot(token, stored.channelUrls.toSet()) + } + return null + } + + suspend fun persist( + userId: String, + selection: SubscriptionSelection, + snapshot: SubscriptionFeedSelectionSnapshot, + ): Boolean { + val token = snapshot.token ?: return true + val channelUrls = snapshot.channelUrls ?: return true + val stored = StoredSubscriptionFeedSelection( + token = token, + filterKey = selection.cursorKey, + channelUrls = channelUrls.sorted(), + ) + val encoded = CacheJson.encodeToString(StoredSubscriptionFeedSelection.serializer(), stored) + for (slot in 0 until MAX_SNAPSHOTS_PER_USER) { + val current = read(userId, slot) + if (current != null) { + if (current == stored) { + val key = SubscriptionFeedCacheKeys.selection(userId, slot) + if (cache.refreshIfValueMatches(key, encoded, SubscriptionFeedSnapshotStore.RETENTION_SECONDS)) { + return true + } + } + continue + } + val key = SubscriptionFeedCacheKeys.selection(userId, slot) + if (cache.setIfAbsent(key, encoded, SubscriptionFeedSnapshotStore.RETENTION_SECONDS)) return true + if (read(userId, slot) == stored) return true + } + return false + } + + private suspend fun read(userId: String, slot: Int): StoredSubscriptionFeedSelection? { + val raw = runCatching { cache.get(SubscriptionFeedCacheKeys.selection(userId, slot)) }.getOrNull() ?: return null - val stored = runCatching { + return runCatching { CacheJson.decodeFromString(StoredSubscriptionFeedSelection.serializer(), raw) - }.getOrNull() ?: return null - if (stored.filterKey != selection.cursorKey) return null - return SubscriptionFeedSelectionSnapshot(token, stored.channelUrls.toSet()) + }.getOrNull() + } + + private fun tokenFor(filterKey: String, channelUrls: Set): String { + val identity = CacheJson.encodeToString( + SubscriptionFeedSelectionIdentity.serializer(), + SubscriptionFeedSelectionIdentity(filterKey, channelUrls.sorted()), + ) + return MessageDigest.getInstance("SHA-256") + .digest(identity.toByteArray()) + .joinToString("") { "%02x".format(it) } + } + + private companion object { + const val MAX_SNAPSHOTS_PER_USER = 8 } } @Serializable private data class StoredSubscriptionFeedSelection( + val token: String, + val filterKey: String, + val channelUrls: List, +) + +@Serializable +private data class SubscriptionFeedSelectionIdentity( val filterKey: String, val channelUrls: List, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 38e75d8d..2bef39f8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -71,18 +71,20 @@ class SubscriptionFeedService( val offset = cursorState?.offset ?: page * limit val selected = selections.resolve(userId, selection, cursorState?.selectionToken) ?: return SubscriptionFeedPageResult.StaleGeneration - return SubscriptionFeedPageResult.Ready( - snapshot.page( - offset, - limit, - isRefreshing(userId), - hideLiveStreams, - hideMembersOnlyContent, - selection, - selected.channelUrls, - selected.token, - ), + val response = snapshot.page( + offset, + limit, + isRefreshing(userId), + hideLiveStreams, + hideMembersOnlyContent, + selection, + selected.channelUrls, + selected.token, ) + if (cursorState == null && response.nextpage != null && !selections.persist(userId, selection, selected)) { + return SubscriptionFeedPageResult.CursorCapacityReached + } + return SubscriptionFeedPageResult.Ready(response) } suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 5d8232a1..e724cf0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -51,7 +51,8 @@ internal object SubscriptionFeedCursorCodec { val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 && - ((it.filterKey == SubscriptionSelection.All.cursorKey) == (it.selectionToken == null)) + ((it.filterKey == SubscriptionSelection.All.cursorKey) == (it.selectionToken == null)) && + (it.selectionToken == null || SELECTION_TOKEN.matches(it.selectionToken)) } ?.let { SubscriptionFeedCursorState( @@ -65,6 +66,8 @@ internal object SubscriptionFeedCursorCodec { ) } }.getOrNull() + + private val SELECTION_TOKEN = Regex("[0-9a-f]{64}") } internal data class SubscriptionFeedCursorState( @@ -137,4 +140,5 @@ internal sealed interface SubscriptionFeedPageResult { data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult data object InvalidCursor : SubscriptionFeedPageResult data object StaleGeneration : SubscriptionFeedPageResult + data object CursorCapacityReached : SubscriptionFeedPageResult } diff --git a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt index ac2319ec..d046ba93 100644 --- a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt +++ b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt @@ -12,6 +12,18 @@ class FakeCacheService : CacheService { values[key] = value } + override suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + values.putIfAbsent(key, value) == null + + override suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean { + var matched = false + values.computeIfPresent(key) { _, current -> + matched = current == value + current + } + return matched + } + override suspend fun delete(key: String) { values.remove(key) } @@ -19,4 +31,6 @@ class FakeCacheService : CacheService { fun clear() { values.clear() } + + fun keys(): Set = values.keys.toSet() } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt new file mode 100644 index 00000000..d528297f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server + +import dev.typetype.server.services.SubscriptionFeedSelectionSnapshot +import dev.typetype.server.services.SubscriptionFeedSelectionStore +import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SubscriptionFeedSelectionStoreTest { + @Test + fun `a ninth distinct session cannot evict the first issued cursor`() = runTest { + val cache = FakeCacheService() + val store = SubscriptionFeedSelectionStore(cache, SubscriptionsService()) + val selections = (1..9).map { index -> + val selection = SubscriptionSelection.Group("group-$index") + val snapshot = SubscriptionFeedSelectionSnapshot( + token = index.toString().padStart(64, '0'), + channelUrls = setOf("https://example.com/channel/$index"), + ) + selection to snapshot + } + + selections.take(8).forEach { (selection, snapshot) -> + assertTrue(store.persist(TEST_USER_ID, selection, snapshot)) + } + val (ninthSelection, ninthSnapshot) = selections.last() + assertFalse(store.persist(TEST_USER_ID, ninthSelection, ninthSnapshot)) + + val (firstSelection, firstSnapshot) = selections.first() + val restored = store.resolve(TEST_USER_ID, firstSelection, firstSnapshot.token) + assertNotNull(restored) + assertEquals(firstSnapshot.channelUrls, restored?.channelUrls) + assertEquals(8, cache.keys().count { it.startsWith("feed:selection") }) + } + + @Test + fun `independent store instances cannot overwrite concurrently issued cursors`() = runTest { + val cache = FakeCacheService() + val stores = List(2) { SubscriptionFeedSelectionStore(cache, SubscriptionsService()) } + val selections = List(2) { index -> + val number = index + 1 + val selection = SubscriptionSelection.Group("group-$number") + val snapshot = SubscriptionFeedSelectionSnapshot( + token = number.toString().padStart(64, '0'), + channelUrls = setOf("https://example.com/channel/$number"), + ) + selection to snapshot + } + val start = CompletableDeferred() + val writes = stores.zip(selections).map { (store, pair) -> + async(Dispatchers.Default) { + start.await() + store.persist(TEST_USER_ID, pair.first, pair.second) + } + } + + start.complete(Unit) + + assertTrue(writes.awaitAll().all { it }) + selections.forEachIndexed { index, (selection, snapshot) -> + assertNotNull(stores[index].resolve(TEST_USER_ID, selection, snapshot.token)) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt index c2becbaa..e4c88e8c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt @@ -26,6 +26,7 @@ import io.mockk.coEvery import io.mockk.mockk import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -35,6 +36,7 @@ class SubscriptionGroupFeedRoutesTest { private val subscriptions = SubscriptionsService() private val groups = SubscriptionGroupsService() private lateinit var feed: SubscriptionFeedService + private lateinit var cache: FakeCacheService private val auth = AuthService.fixed(TEST_USER_ID) companion object { @@ -46,7 +48,8 @@ class SubscriptionGroupFeedRoutesTest { @BeforeEach fun clean() { TestDatabase.truncateAll() - feed = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + cache = FakeCacheService() + feed = SubscriptionFeedService(subscriptions, FakeChannelService(), cache) } private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { @@ -87,7 +90,7 @@ class SubscriptionGroupFeedRoutesTest { coEvery { channelService.getChannel(channel("three"), null) } returns SubscriptionFeedTestFixtures.channel( SubscriptionFeedTestFixtures.video(1_000L, channel = "three", url = "video-three"), ) - feed = SubscriptionFeedService(subscriptions, channelService, FakeCacheService()) + feed = SubscriptionFeedService(subscriptions, channelService, cache) listOf("one", "two", "three").forEach { subscriptions.add(TEST_USER_ID, subscription(it)) } val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group groups.addSubscription(TEST_USER_ID, group.id, channel("one")) @@ -96,15 +99,34 @@ class SubscriptionGroupFeedRoutesTest { feed.awaitRefresh(TEST_USER_ID) val firstPage = requestReadyFeed(limit = 1, groupId = group.id) assertEquals(listOf("video-one"), firstPage.videos.map { it.url }) + val repeatedFirstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertEquals(firstPage.nextpage, repeatedFirstPage.nextpage) + assertEquals(1, cache.keys().count { it.startsWith("feed:selection") }) groups.removeSubscription(TEST_USER_ID, group.id, channel("two")) groups.addSubscription(TEST_USER_ID, group.id, channel("three")) + val changedFirstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertNotEquals(firstPage.nextpage, changedFirstPage.nextpage) + assertEquals(2, cache.keys().count { it.startsWith("feed:selection") }) val secondPage = requestFeed(limit = 1, cursor = requireNotNull(firstPage.nextpage), groupId = group.id) assertEquals(HttpStatusCode.OK, secondPage.status) assertEquals(listOf("video-two"), Json.decodeFromString(secondPage.bodyAsText()).videos.map { it.url }) } + @Test + fun `terminal filtered page does not retain a membership snapshot`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + + assertTrue(cache.keys().none { it.startsWith("feed:selection") }) + } + @Test fun `cursor cannot be reused with another subscription filter`() = withApp { subscriptions.add(TEST_USER_ID, subscription("one")) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt index df2da9ac..96097f40 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt @@ -19,14 +19,12 @@ import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import java.nio.file.Files -import java.nio.file.Path class SubscriptionsRoutesTest { @@ -44,7 +42,7 @@ class SubscriptionsRoutesTest { private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { - install(ContentNegotiation) { json() } + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } routing { subscriptionsRoutes(service, auth) } } block() @@ -52,19 +50,6 @@ class SubscriptionsRoutesTest { private val itemBody = """{"channelUrl":"https://yt.com/channel/1","name":"Test","avatarUrl":""}""" - @Test - fun `subscription creation contract omits the server timestamp`() { - val components = Files.readString(Path.of("openapi/components/subscriptions.yaml")) - val requestSchema = components - .substringAfter("SubscriptionCreateRequest:") - .substringBefore("SubscriptionGroupItem:") - val paths = Files.readString(Path.of("openapi/paths/subscriptions.yaml")) - - assertTrue("required: [channelUrl, name, avatarUrl]" in requestSchema) - assertFalse("subscribedAt" in requestSchema) - assertTrue("#/SubscriptionCreateRequest" in paths) - } - @Test fun `GET subscriptions without token returns 401`() = withApp { assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions").status) @@ -78,14 +63,31 @@ class SubscriptionsRoutesTest { } @Test - fun `POST subscriptions returns 201 and persists item`() = withApp { + fun `POST subscriptions generates and persists the server timestamp`() = withApp { val response = client.post("/subscriptions") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) setBody(itemBody) } assertEquals(HttpStatusCode.Created, response.status) - assertTrue(response.bodyAsText().contains("\"channelUrl\":\"https://yt.com/channel/1\"")) + val created = Json.decodeFromString(response.bodyAsText()) + assertEquals("https://yt.com/channel/1", created.channelUrl) + assertTrue(created.subscribedAt > 1) + assertEquals(created.subscribedAt, service.getAll(TEST_USER_ID).single().subscribedAt) + } + + @Test + fun `POST subscriptions ignores the obsolete client timestamp`() = withApp { + val response = client.post("/subscriptions") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody(itemBody.dropLast(1) + ",\"subscribedAt\":1}") + } + + assertEquals(HttpStatusCode.Created, response.status) + val created = Json.decodeFromString(response.bodyAsText()) + assertTrue(created.subscribedAt > 1) + assertEquals(created.subscribedAt, service.getAll(TEST_USER_ID).single().subscribedAt) } @Test diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt index 7d9a2219..1d9a7d8f 100644 --- a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt @@ -152,6 +152,31 @@ class TypeTypeBackupServiceTest { assertTrue(backup.history == null) } + @Test + fun `legacy subscription backup without groups preserves compatible memberships`() = runTest { + subscriptions.add(TARGET, SubscriptionItem("https://youtube.com/channel/keep", "Keep", "")) + subscriptions.add(TARGET, SubscriptionItem("https://youtube.com/channel/drop", "Drop", "")) + val group = (subscriptionGroups.create(TARGET, "Existing") as SubscriptionGroupWriteResult.Success).group + subscriptionGroups.addSubscription(TARGET, group.id, "https://youtube.com/channel/keep") + subscriptionGroups.addSubscription(TARGET, group.id, "https://youtube.com/channel/drop") + val legacyBackup = TypeTypeBackupItem( + exportedAt = 1, + categories = listOf(TypeTypeBackupCategory.SUBSCRIPTIONS.wireName), + subscriptions = listOf( + SubscriptionItem("https://youtube.com/channel/keep", "Keep", "", subscribedAt = 1), + ), + ) + + service.restore(TARGET, legacyBackup) + + val preservedGroup = subscriptionGroups.getAll(TARGET).single() + assertEquals(group.id, preservedGroup.id) + assertEquals( + listOf("https://youtube.com/channel/keep"), + subscriptionGroups.getChannelUrls(TARGET, preservedGroup.id), + ) + } + @Test fun `restore rejects empty normalized blocked keywords`() = runTest { val backup = TypeTypeBackupItem( From 93a0830aae597477c51c85f9248fdde93c350aab Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 19:46:54 -0700 Subject: [PATCH 48/65] fix: prevent orphaned subscription memberships Subscription group assignment checked subscription ownership without coordinating with unsubscribe or replacement restores. Take an account-keyed transaction advisory lock across membership assignment and every subscription removal or replacement path so the check and insert cannot straddle a committed deletion. Constraint: Replacement imports must retain memberships whose subscriptions survive the import Rejected: Composite foreign key with cascading deletes | cascade semantics would discard memberships before retained subscriptions are reinserted Confidence: high Scope-risk: narrow Directive: Any new path that removes or replaces an account's subscriptions must acquire SubscriptionMutationLock in the same transaction Tested: Focused PostgreSQL concurrency regression on JDK 25 Not-tested: Full suite and live HTTP concurrency gate run after both fix commits --- .../PipePipeBackupPersisterService.kt | 1 + .../services/SubscriptionGroupsService.kt | 1 + .../services/SubscriptionMutationLock.kt | 14 +++ .../server/services/SubscriptionsService.kt | 1 + .../services/TypeTypeBackupRestoreWriter.kt | 1 + .../server/SubscriptionGroupsServiceTest.kt | 89 +++++++++++++++++++ 6 files changed, 107 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt index 4e965014..4ee60d97 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt @@ -16,6 +16,7 @@ import java.util.UUID class PipePipeBackupPersisterService { suspend fun persist(userId: String, snapshot: PipePipeBackupSnapshotItem): PipePipeBackupRestoreResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) clearUserData(userId) val avatarsByChannel = snapshot.subscriptions .mapNotNull { item -> item.url.takeIf { it.isNotBlank() }?.let { url -> url to item.avatarUrl } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 23887b2c..2566c5ed 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -101,6 +101,7 @@ class SubscriptionGroupsService { groupId: String, rawChannelUrl: String, ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) val subscriptionExists = SubscriptionsTable.selectAll().where { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt new file mode 100644 index 00000000..65caa657 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.services + +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager + +internal object SubscriptionMutationLock { + fun acquire(userId: String) { + val userKey = userId.hashCode() and Int.MAX_VALUE + TransactionManager.current().exec( + "SELECT pg_advisory_xact_lock($LOCK_NAMESPACE, $userKey)", + ) + } + + private const val LOCK_NAMESPACE = 1_414_814_032 +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 21d022de..6e90ab8f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -46,6 +46,7 @@ class SubscriptionsService { } suspend fun delete(userId: String, channelUrl: String): Boolean = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(channelUrl) SubscriptionGroupMembershipsTable.deleteWhere { (SubscriptionGroupMembershipsTable.userId eq userId) and diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt index d951dafd..b8209cb9 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -10,6 +10,7 @@ internal object TypeTypeBackupRestoreWriter { backup: TypeTypeBackupItem, categories: Set, ): TypeTypeRestoreSummary = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val restored = linkedMapOf() if (TypeTypeBackupCategory.SUBSCRIPTIONS in categories) { restored["subscriptions"] = TypeTypeBackupCoreRestore.subscriptions( diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index d03bce44..74ba5762 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.TypeTypeBackupItem import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.services.SubscriptionGroupMembershipCleaner import dev.typetype.server.services.SubscriptionGroupMembershipResult @@ -8,13 +9,23 @@ import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupRestoreWriter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.yield +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit class SubscriptionGroupsServiceTest { private val groups = SubscriptionGroupsService() @@ -98,6 +109,61 @@ class SubscriptionGroupsServiceTest { assertEquals(emptyList(), groups.getChannelUrls("user", group.id)) } + @Test + fun `membership assignment deletion and replacement share a user lock`() = runTest { + val userId = "concurrent-user" + val group = groups.create(userId, "Group").createdGroup() + subscriptions.add(userId, subscription("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val assignment = async(Dispatchers.IO) { + groups.addSubscription(userId, group.id, channel("one")) + } + val deletion = async(Dispatchers.IO) { subscriptions.delete(userId, channel("one")) } + val replacement = async(Dispatchers.IO) { + TypeTypeBackupRestoreWriter.restore( + userId = userId, + backup = TypeTypeBackupItem( + exportedAt = 1, + categories = listOf(TypeTypeBackupCategory.SUBSCRIPTIONS.wireName), + subscriptions = listOf(subscription("one").copy(subscribedAt = 1)), + ), + categories = setOf(TypeTypeBackupCategory.SUBSCRIPTIONS), + ) + } + val allWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + var waiting = false + while (!waiting && !(assignment.isCompleted && deletion.isCompleted && replacement.isCompleted)) { + waiting = waitingSubscriptionLocks(userId) >= 3 + if (!waiting) yield() + } + waiting + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assignment.await() + assertTrue(deletion.await()) + replacement.await() + assertTrue(allWaited, "all mutations must wait for the same account-scoped lock") + val subscriptionUrls = subscriptions.getAll(userId).mapTo(hashSetOf(), SubscriptionItem::channelUrl) + assertTrue(groups.getChannelUrls(userId, group.id).all { it in subscriptionUrls }) + } + @Test fun `replacement imports retain only memberships for subscriptions still present`() = runTest { val group = groups.create("user", "Group").createdGroup() @@ -117,4 +183,27 @@ class SubscriptionGroupsServiceTest { private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") private fun channel(id: String) = "https://yt.com/channel/$id" + + private fun subscriptionLockSql(userId: String): String = + "SELECT pg_advisory_xact_lock($SUBSCRIPTION_LOCK_NAMESPACE, ${subscriptionLockKey(userId)})" + + private suspend fun waitingSubscriptionLocks(userId: String): Int = DatabaseFactory.query { + TransactionManager.current().exec( + """ + SELECT count(*) + FROM pg_locks + WHERE locktype = 'advisory' + AND classid = $SUBSCRIPTION_LOCK_NAMESPACE + AND objid = ${subscriptionLockKey(userId)} + AND NOT granted + """.trimIndent(), + ) { result -> + result.next() + result.getInt(1) + } ?: 0 + } + + private fun subscriptionLockKey(userId: String): Int = userId.hashCode() and Int.MAX_VALUE } + +private const val SUBSCRIPTION_LOCK_NAMESPACE = 1_414_814_032 From 267fda9a7cf7854655b4289f9cf7665c33f26c2f Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 19:47:08 -0700 Subject: [PATCH 49/65] fix: keep exported subscription backups restorable Subscription and group sections were read in separate transactions, so a membership committed between them could reference a subscription absent from the exported list. Capture subscriptions once and export only group memberships belonging to that captured set, preserving the restore validator's referential invariant. Constraint: Subscription groups remain coupled to the subscriptions backup category Rejected: Add a cross-service export transaction | existing service reads open their own transactions and captured-set filtering is the smaller accepted repair Confidence: high Scope-risk: narrow Directive: Exported group memberships must remain a subset of the subscriptions captured for the same backup Tested: Focused mixed-read export and restore regression on JDK 25 Not-tested: Full suite and live HTTP concurrency gate run after this commit --- .../SubscriptionGroupBackupRepository.kt | 8 ++- .../server/services/TypeTypeBackupService.kt | 16 +++-- .../SubscriptionBackupConsistencyTest.kt | 72 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt index 3cc685aa..3df60b25 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt @@ -13,10 +13,16 @@ import java.util.Locale import java.util.UUID internal object SubscriptionGroupBackupRepository { - suspend fun export(userId: String): List = DatabaseFactory.query { + suspend fun export( + userId: String, + subscriptionUrls: Set, + ): List = DatabaseFactory.query { val channelsByGroup = SubscriptionGroupMembershipsTable.selectAll() .where { SubscriptionGroupMembershipsTable.userId eq userId } .orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.ASC) + .filter { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionGroupMembershipsTable.channelUrl]) in subscriptionUrls + } .groupBy( keySelector = { it[SubscriptionGroupMembershipsTable.groupId] }, valueTransform = { it[SubscriptionGroupMembershipsTable.channelUrl] }, diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt index 97466980..fb341ae2 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt @@ -31,14 +31,20 @@ class TypeTypeBackupService( } else { null } + val subscriptionItems = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) { + subscriptions.getAll(userId) + } else { + null + } return TypeTypeBackupItem( exportedAt = System.currentTimeMillis(), categories = categories.map(TypeTypeBackupCategory::wireName).sorted(), - subscriptions = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) subscriptions.getAll(userId) else null, - subscriptionGroups = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) { - SubscriptionGroupBackupRepository.export(userId) - } else { - null + subscriptions = subscriptionItems, + subscriptionGroups = subscriptionItems?.let { items -> + val channelUrls = items.mapTo(hashSetOf()) { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) + } + SubscriptionGroupBackupRepository.export(userId, channelUrls) }, history = if (includes(TypeTypeBackupCategory.HISTORY)) history.getAll(userId) else null, playlists = fullPlaylists, diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt new file mode 100644 index 00000000..02afdec5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt @@ -0,0 +1,72 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupService +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionBackupConsistencyTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `backup stays restorable when a subscription is added between section reads`() = runTest { + val group = (groups.create(SOURCE, "New") as SubscriptionGroupWriteResult.Success).group + val capturedSubscriptions = mockk() + coEvery { capturedSubscriptions.getAll(SOURCE, any()) } coAnswers { + subscriptions.add(SOURCE, SubscriptionItem(CHANNEL_URL, "Channel", "")) + assertEquals( + SubscriptionGroupMembershipResult.Success, + groups.addSubscription(SOURCE, group.id, CHANNEL_URL), + ) + emptyList() + } + val service = backupService(capturedSubscriptions) + + val backup = service.export(SOURCE, setOf(TypeTypeBackupCategory.SUBSCRIPTIONS)) + val restored = service.restore(TARGET, backup) + + assertEquals(emptyList(), backup.subscriptions) + assertEquals(emptyList(), backup.subscriptionGroups?.single()?.channelUrls) + assertEquals(1, restored.restored["subscriptionGroups"]) + assertEquals(0, restored.restored["subscriptionGroupMemberships"]) + } + + private fun backupService(subscriptions: SubscriptionsService) = TypeTypeBackupService( + subscriptions = subscriptions, + history = mockk(), + playlists = mockk(), + watchLater = mockk(), + favorites = mockk(), + progress = mockk(), + searchHistory = mockk(), + savedPlaylists = mockk(), + settings = mockk(), + blocked = mockk(), + allowedChannels = mockk(), + allowedPlaylists = mockk(), + ) +} + +private const val SOURCE = "concurrent-backup-source" +private const val TARGET = "concurrent-backup-target" +private const val CHANNEL_URL = "https://youtube.com/channel/concurrent" From af9fb57b8ee108071f45ae775562bef2d2f7839c Mon Sep 17 00:00:00 2001 From: User Date: Mon, 17 Aug 2026 22:01:16 -0700 Subject: [PATCH 50/65] refactor: clarify subscription lock namespace The previous decimal constant encoded an undocumented product tag. Use the precomputed PostgreSQL hashtext value for the literal subscriptions namespace and document its origin while retaining numeric lock lookup at runtime. Constraint: Do not evaluate hashtext on every lock acquisition Rejected: Runtime hashtext('subscriptions') | repeats derivation on every lock lookup Confidence: high Scope-risk: narrow Directive: Keep the production and concurrency-test namespace constants identical Tested: Focused PostgreSQL subscription mutation concurrency regression on JDK 25 Not-tested: Full suite because the change only replaces one fixed namespace key consistently --- .../dev/typetype/server/services/SubscriptionMutationLock.kt | 3 ++- .../dev/typetype/server/SubscriptionGroupsServiceTest.kt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt index 65caa657..8320b1ab 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -10,5 +10,6 @@ internal object SubscriptionMutationLock { ) } - private const val LOCK_NAMESPACE = 1_414_814_032 + // Precomputed PostgreSQL hashtext('subscriptions'). + private const val LOCK_NAMESPACE = 720_815_616 } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index 74ba5762..f0bcfaca 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -206,4 +206,5 @@ class SubscriptionGroupsServiceTest { private fun subscriptionLockKey(userId: String): Int = userId.hashCode() and Int.MAX_VALUE } -private const val SUBSCRIPTION_LOCK_NAMESPACE = 1_414_814_032 +// Precomputed PostgreSQL hashtext('subscriptions'); must match SubscriptionMutationLock. +private const val SUBSCRIPTION_LOCK_NAMESPACE = 720_815_616 From 4fc80814078490e6a7c8d4ba4f919b92aaabbc33 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 12:50:27 +0200 Subject: [PATCH 51/65] fix: hide finished live streams from feeds --- .../typetype/server/services/SubscriptionFeedSnapshot.kt | 2 +- .../dev/typetype/server/services/VideoItemSchedule.kt | 3 +++ .../server/SubscriptionFeedLiveVisibilityRoutesTest.kt | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index c752034a..cc53b5a7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -71,7 +71,7 @@ internal fun SubscriptionFeedSnapshot.page( hideMembersOnlyContent: Boolean = false, ): SubscriptionFeedResponse { val visibleVideos = videos.filterNot { video -> - (hideLiveStreams && video.isLiveOrUpcomingAt(generatedAt)) || + (hideLiveStreams && video.isLiveContentOrUpcomingAt(generatedAt)) || (hideMembersOnlyContent && video.requiresMembership) } val from = offset.coerceAtMost(visibleVideos.size) diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt index 25782930..e8f1837f 100644 --- a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt +++ b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -6,3 +6,6 @@ internal fun VideoItem.isUpcomingAt(now: Long): Boolean = !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) + +internal fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = + isLive || isPostLive || isLiveContent || isUpcomingAt(now) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index c2c0ec10..3dee3689 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -87,15 +87,15 @@ class SubscriptionFeedLiveVisibilityRoutesTest { } @Test - fun `hidden live streams do not remove finished recordings`() = withApp { + fun `account setting hides finished live recordings`() = withApp { val channelService = mockk() coEvery { channelService.getChannel(any(), null) } returns channel( - video(4_000L, url = "https://youtube.com/watch?v=live", live = true), video(3_000L, url = "https://youtube.com/watch?v=replay").copy( streamType = "post_live_stream", isPostLive = true, - isLiveContent = true, ), + video(2_500L, url = "https://youtube.com/watch?v=live-content").copy(isLiveContent = true), + video(2_000L, url = "https://youtube.com/watch?v=normal"), ) feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) subscriptionsService.add(TEST_USER_ID, subscription(1)) @@ -104,7 +104,7 @@ class SubscriptionFeedLiveVisibilityRoutesTest { assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) feedService.awaitRefresh(TEST_USER_ID) - assertEquals(listOf("replay"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) } @Test From ad2073dd3d28e603c3b699ae3e5ff1b3b9b08cdd Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 13:18:19 +0200 Subject: [PATCH 52/65] fix: classify streams tab feed entries --- .../services/SubscriptionFeedBuilder.kt | 6 +++-- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index bae327dc..0af08a4a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -42,7 +42,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic val videos = if (liveResult == null) { channelResult.videos } else { - channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos + channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos.map { it.asLiveContent() } } val results = listOfNotNull(channelResult, liveResult) SubscriptionSourceResult( @@ -73,10 +73,12 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic this@deduplicated.forEach { video -> val key = video.subscriptionFeedKey() val current = get(key) - if (current == null || video.isLive && !current.isLive) put(key, video) + if (current == null || video.isLiveContent && !current.isLiveContent) put(key, video) } }.values.toList() + private fun VideoItem.asLiveContent(): VideoItem = if (isLiveContent) this else copy(isLiveContent = true) + private fun String.toLivestreamsTabUrl(): String { val uri = URI(this) val path = uri.path.trimEnd('/') diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index 3dee3689..15a357b3 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -107,6 +107,28 @@ class SubscriptionFeedLiveVisibilityRoutesTest { assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) } + @Test + fun `account setting hides unclassified videos from streams tab`() = withApp { + val channelUrl = "https://www.youtube.com/channel/UC1" + val channelService = mockk() + coEvery { channelService.getChannel(channelUrl, null) } returns channel( + video(3_000L, url = "https://youtube.com/watch?v=normal"), + video(2_000L, url = "https://youtube.com/watch?v=replay"), + ) + coEvery { channelService.getChannel("$channelUrl/streams", null) } returns channel( + video(2_000L, url = "https://youtube.com/watch?v=replay"), + video(1_000L, url = "https://youtube.com/watch?v=scheduled"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + @Test fun `account setting hides members only videos before pagination`() = withApp { val channelService = mockk() From 628dab4557aaeac608f07d43f0b5e4a77f36e71d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 14:04:11 +0200 Subject: [PATCH 53/65] fix: keep auth database work off request threads --- .../typetype/server/UserDataRateLimitKey.kt | 4 +- .../dev/typetype/server/routes/AuthRoutes.kt | 2 +- .../dev/typetype/server/routes/UserAuth.kt | 2 +- .../typetype/server/services/AuthService.kt | 50 +++++++++++-------- .../server/services/OidcUserService.kt | 2 +- .../typetype/server/AuthServiceCoreTest.kt | 11 ++-- .../server/AuthServiceDispatcherTest.kt | 47 +++++++++++++++++ .../server/PasswordResetServiceCoreTest.kt | 5 +- 8 files changed, 91 insertions(+), 32 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt diff --git a/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt b/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt index 454edc98..e5720ec9 100644 --- a/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt +++ b/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt @@ -3,11 +3,11 @@ package dev.typetype.server import dev.typetype.server.services.AuthService import io.ktor.server.application.ApplicationCall -fun userDataRateLimitKey(call: ApplicationCall, authService: AuthService): String { +suspend fun userDataRateLimitKey(call: ApplicationCall, authService: AuthService): String { val bearerToken = call.request.headers["Authorization"] ?.takeIf { it.startsWith("Bearer ") } ?.substringAfter("Bearer ") - val userId = bearerToken?.let(authService::verify) + val userId = bearerToken?.let { authService.verify(it) } if (userId != null) return "user:$userId" return "ip:${call.request.headers["X-Real-IP"] ?: call.request.local.remoteHost}" } diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt index 19bfda0f..d791eeaf 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt @@ -114,5 +114,5 @@ fun Route.authRoutes( } } -private fun String.warm(authService: AuthService, warmupService: HomeRecommendationWarmup): Unit = +private suspend fun String.warm(authService: AuthService, warmupService: HomeRecommendationWarmup): Unit = authService.verify(this)?.let(warmupService::markActive) ?: Unit diff --git a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt b/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt index 5970649f..6aaf8259 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt @@ -21,7 +21,7 @@ suspend fun ApplicationCall.withJwtAuth(authService: AuthService, block: suspend block(userId) } -fun ApplicationCall.optionalJwtUserId(authService: AuthService): String? { +suspend fun ApplicationCall.optionalJwtUserId(authService: AuthService): String? { val authHeader = request.headers["Authorization"] if (authHeader == null || !authHeader.startsWith("Bearer ")) return null val token = authHeader.substringAfter("Bearer ") diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 21a5eb59..f0e09374 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -4,6 +4,8 @@ import com.auth0.jwt.JWT import com.auth0.jwt.algorithms.Algorithm import com.password4j.Password import dev.typetype.server.db.tables.UsersTable +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.lowerCase import org.jetbrains.exposed.v1.jdbc.insert @@ -24,12 +26,12 @@ open class AuthService( private val sessionVerifier = AuthSessionVerifier(accessCodec, sessionStore) private val sessionRevoker = AuthSessionRevoker(sessionStore) - fun register(email: String, password: String, name: String): AuthSessionTokens { + suspend fun register(email: String, password: String, name: String): AuthSessionTokens = withContext(Dispatchers.IO) { val hashed = Password.hash(password).withArgon2().result val userId = UUID.randomUUID().toString() val now = System.currentTimeMillis() - val needsAdmin = !hasAdmin() + val needsAdmin = !hasAdminBlocking() val role = if (needsAdmin) "admin" else "user" val publicUsername = name.trim().takeIf(ProfileService::isValidPublicUsername) @@ -45,12 +47,12 @@ open class AuthService( it[UsersTable.updatedAt] = now } } - return tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") + tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") } - fun login(identifier: String, password: String): AuthSessionTokens? { + suspend fun login(identifier: String, password: String): AuthSessionTokens? = withContext(Dispatchers.IO) { val normalizedIdentifier = identifier.trim().lowercase() - if (normalizedIdentifier.isBlank()) return null + if (normalizedIdentifier.isBlank()) return@withContext null val user = transaction { val query = UsersTable.selectAll().where { if (normalizedIdentifier.contains("@")) { @@ -60,25 +62,29 @@ open class AuthService( } } query.singleOrNull() - } ?: return null + } ?: return@withContext null val hashed = user[UsersTable.passwordHash] val verified = Password.check(password, hashed).withArgon2() - if (!verified) return null + if (!verified) return@withContext null - return tokenIssuer.issue(user[UsersTable.id]) + tokenIssuer.issue(user[UsersTable.id]) } - fun refreshSession(refreshToken: String): AuthSessionTokens? = sessionRefresher.refresh(refreshToken) + suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = withContext(Dispatchers.IO) { + sessionRefresher.refresh(refreshToken) + } - fun issueSession(userId: String): AuthSessionTokens? = tokenIssuer.issue(userId) + suspend fun issueSession(userId: String): AuthSessionTokens? = withContext(Dispatchers.IO) { + tokenIssuer.issue(userId) + } - fun logout(refreshToken: String?) { + suspend fun logout(refreshToken: String?): Unit = withContext(Dispatchers.IO) { sessionRevoker.revokeByRefreshToken(refreshToken) } - open fun verify(token: String): String? { - return sessionVerifier.verifyUserId(token) + open suspend fun verify(token: String): String? = withContext(Dispatchers.IO) { + sessionVerifier.verifyUserId(token) } fun guestLogin(): String { @@ -91,16 +97,20 @@ open class AuthService( .sign(Algorithm.HMAC256(jwtSecret)) } - fun getUserRole(userId: String): String? { - if (userId.startsWith("guest:")) return "user" - return transaction { + suspend fun getUserRole(userId: String): String? = withContext(Dispatchers.IO) { + if (userId.startsWith("guest:")) return@withContext "user" + transaction { UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull() }?.get(UsersTable.role) } - fun hasUsers(): Boolean = hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().empty().not() } + suspend fun hasUsers(): Boolean = withContext(Dispatchers.IO) { + hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().empty().not() } + } + + suspend fun hasAdmin(): Boolean = withContext(Dispatchers.IO) { hasAdminBlocking() } - fun hasAdmin(): Boolean = hasUsersProbe?.invoke() ?: transaction { + private fun hasAdminBlocking(): Boolean = hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().where { UsersTable.role eq "admin" }.empty().not() } @@ -108,11 +118,11 @@ open class AuthService( private const val GUEST_TTL_MS = 7 * 24 * 60 * 60 * 1000L fun fixed(userId: String): AuthService = object : AuthService("test") { - override fun verify(token: String): String? = if (token == "test-jwt") userId else null + override suspend fun verify(token: String): String? = if (token == "test-jwt") userId else null } fun fixed(userId: String, hasUsers: Boolean): AuthService = object : AuthService("test", { hasUsers }) { - override fun verify(token: String): String? = if (token == "test-jwt") userId else null + override suspend fun verify(token: String): String? = if (token == "test-jwt") userId else null } } } diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt index 920031e0..d6c50d06 100644 --- a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.jdbc.update import java.util.UUID class OidcUserService(private val authService: AuthService) { - fun login(identity: OidcIdentity): AuthSessionTokens { + suspend fun login(identity: OidcIdentity): AuthSessionTokens { val userId = transaction { resolveUserId(identity) } return authService.issueSession(userId) ?: throw IllegalStateException("Failed to create session") } diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt index 487f90fb..56741b55 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.AuthSessionConfig +import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -30,7 +31,7 @@ class AuthServiceCoreTest { } @Test - fun `register sets first admin and second user`() { + fun `register sets first admin and second user`() = runTest { val service = AuthService("test-secret") assertFalse(service.hasUsers()) assertFalse(service.hasAdmin()) @@ -63,7 +64,7 @@ class AuthServiceCoreTest { } @Test - fun `login and refresh token keep same user`() { + fun `login and refresh token keep same user`() = runTest { val service = AuthService("test-secret") val registered = service.register("login@test.local", "secret-1", "Login") val expectedUser = service.verify(registered.accessToken) @@ -79,7 +80,7 @@ class AuthServiceCoreTest { } @Test - fun `configured refresh lifetime is stored for new sessions`() { + fun `configured refresh lifetime is stored for new sessions`() = runTest { val before = System.currentTimeMillis() val service = AuthService( "test-secret", @@ -96,7 +97,7 @@ class AuthServiceCoreTest { } @Test - fun `login supports public username identifier`() { + fun `login supports public username identifier`() = runTest { val service = AuthService("test-secret") val session = service.register("username@test.local", "secret-1", "User") val userId = service.verify(session.accessToken) ?: error("missing user id") @@ -111,7 +112,7 @@ class AuthServiceCoreTest { } @Test - fun `guest token verifies and has user role`() { + fun `guest token verifies and has user role`() = runTest { val service = AuthService("test-secret") val guestToken = service.guestLogin() val guestId = service.verify(guestToken) diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt new file mode 100644 index 00000000..08d69779 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt @@ -0,0 +1,47 @@ +package dev.typetype.server + +import dev.typetype.server.services.AuthService +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class AuthServiceDispatcherTest { + @Test + fun `slow database work does not block the caller dispatcher`() { + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + val callerDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + val service = AuthService("test-secret", hasUsersProbe = { + entered.countDown() + release.await() + false + }) + + try { + runBlocking { + val pending = async(callerDispatcher) { service.hasUsers() } + assertTrue(entered.await(2, TimeUnit.SECONDS)) + + val result = withTimeout(1_000) { + withContext(callerDispatcher) { "responsive" } + } + assertEquals("responsive", result) + + release.countDown() + assertFalse(pending.await()) + } + } finally { + release.countDown() + callerDispatcher.close() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt index 98539eb1..8f0af090 100644 --- a/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.PasswordResetService +import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update @@ -27,7 +28,7 @@ class PasswordResetServiceCoreTest { } @Test - fun `reset password updates credentials and token cannot be reused`() { + fun `reset password updates credentials and token cannot be reused`() = runTest { val auth = AuthService("test-secret") val reset = PasswordResetService() val oldPassword = "secret-1" @@ -43,7 +44,7 @@ class PasswordResetServiceCoreTest { } @Test - fun `expired token is rejected`() { + fun `expired token is rejected`() = runTest { val auth = AuthService("test-secret") val reset = PasswordResetService() val userId = auth.verify(auth.register("expired@test.local", "secret-1", "Expired").accessToken) From 4be753c2a7842f01640f663b33f47098ca6b7b4e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 15:10:21 +0200 Subject: [PATCH 54/65] fix: isolate database work from shared IO --- .../dev/typetype/server/db/DatabaseFactory.kt | 10 +++- .../typetype/server/services/AuthService.kt | 46 ++++++++++--------- .../server/AuthServiceDispatcherTest.kt | 27 +++++++++++ 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 0da240a5..a9fb5d3b 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -38,6 +38,9 @@ import org.jetbrains.exposed.v1.jdbc.SchemaUtils import org.jetbrains.exposed.v1.jdbc.transactions.transaction object DatabaseFactory { + private const val POOL_SIZE = 10 + private val queryDispatcher = Dispatchers.IO.limitedParallelism(POOL_SIZE, "database") + fun init(url: String, user: String, password: String) { val dbPassword = password val config = HikariConfig().apply { @@ -45,7 +48,7 @@ object DatabaseFactory { username = user this.password = dbPassword driverClassName = "org.postgresql.Driver" - maximumPoolSize = 10 + maximumPoolSize = POOL_SIZE minimumIdle = 2 } Database.connect(HikariDataSource(config)) @@ -132,7 +135,10 @@ object DatabaseFactory { DatabaseCollectionMetadataMigration.apply() } } - suspend fun query(block: () -> T): T = withContext(Dispatchers.IO) { transaction { block() } } + suspend fun query(block: () -> T): T = blocking { transaction { block() } } + + suspend fun blocking(block: () -> T): T = withContext(queryDispatcher) { block() } + fun healthCheck(): Boolean = runCatching { transaction { exec("SELECT 1") { it.next() } == true } }.getOrDefault(false) diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index f0e09374..54a81a8e 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import com.auth0.jwt.JWT import com.auth0.jwt.algorithms.Algorithm import com.password4j.Password +import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.UsersTable import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -26,16 +27,16 @@ open class AuthService( private val sessionVerifier = AuthSessionVerifier(accessCodec, sessionStore) private val sessionRevoker = AuthSessionRevoker(sessionStore) - suspend fun register(email: String, password: String, name: String): AuthSessionTokens = withContext(Dispatchers.IO) { - val hashed = Password.hash(password).withArgon2().result + suspend fun register(email: String, password: String, name: String): AuthSessionTokens { + val hashed = withContext(passwordDispatcher) { Password.hash(password).withArgon2().result } val userId = UUID.randomUUID().toString() val now = System.currentTimeMillis() - val needsAdmin = !hasAdminBlocking() + val needsAdmin = !hasAdmin() val role = if (needsAdmin) "admin" else "user" val publicUsername = name.trim().takeIf(ProfileService::isValidPublicUsername) - transaction { + DatabaseFactory.query { UsersTable.insert { it[UsersTable.id] = userId it[UsersTable.email] = email @@ -47,13 +48,15 @@ open class AuthService( it[UsersTable.updatedAt] = now } } - tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") + return DatabaseFactory.blocking { + tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") + } } - suspend fun login(identifier: String, password: String): AuthSessionTokens? = withContext(Dispatchers.IO) { + suspend fun login(identifier: String, password: String): AuthSessionTokens? { val normalizedIdentifier = identifier.trim().lowercase() - if (normalizedIdentifier.isBlank()) return@withContext null - val user = transaction { + if (normalizedIdentifier.isBlank()) return null + val user = DatabaseFactory.query { val query = UsersTable.selectAll().where { if (normalizedIdentifier.contains("@")) { UsersTable.email.lowerCase() eq normalizedIdentifier @@ -62,28 +65,28 @@ open class AuthService( } } query.singleOrNull() - } ?: return@withContext null + } ?: return null val hashed = user[UsersTable.passwordHash] - val verified = Password.check(password, hashed).withArgon2() - if (!verified) return@withContext null + val verified = withContext(passwordDispatcher) { Password.check(password, hashed).withArgon2() } + if (!verified) return null - tokenIssuer.issue(user[UsersTable.id]) + return DatabaseFactory.blocking { tokenIssuer.issue(user[UsersTable.id]) } } - suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = withContext(Dispatchers.IO) { + suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = DatabaseFactory.blocking { sessionRefresher.refresh(refreshToken) } - suspend fun issueSession(userId: String): AuthSessionTokens? = withContext(Dispatchers.IO) { + suspend fun issueSession(userId: String): AuthSessionTokens? = DatabaseFactory.blocking { tokenIssuer.issue(userId) } - suspend fun logout(refreshToken: String?): Unit = withContext(Dispatchers.IO) { + suspend fun logout(refreshToken: String?): Unit = DatabaseFactory.blocking { sessionRevoker.revokeByRefreshToken(refreshToken) } - open suspend fun verify(token: String): String? = withContext(Dispatchers.IO) { + open suspend fun verify(token: String): String? = DatabaseFactory.blocking { sessionVerifier.verifyUserId(token) } @@ -97,18 +100,18 @@ open class AuthService( .sign(Algorithm.HMAC256(jwtSecret)) } - suspend fun getUserRole(userId: String): String? = withContext(Dispatchers.IO) { - if (userId.startsWith("guest:")) return@withContext "user" - transaction { + suspend fun getUserRole(userId: String): String? { + if (userId.startsWith("guest:")) return "user" + return DatabaseFactory.query { UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull() }?.get(UsersTable.role) } - suspend fun hasUsers(): Boolean = withContext(Dispatchers.IO) { + suspend fun hasUsers(): Boolean = DatabaseFactory.blocking { hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().empty().not() } } - suspend fun hasAdmin(): Boolean = withContext(Dispatchers.IO) { hasAdminBlocking() } + suspend fun hasAdmin(): Boolean = DatabaseFactory.blocking { hasAdminBlocking() } private fun hasAdminBlocking(): Boolean = hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().where { UsersTable.role eq "admin" }.empty().not() @@ -116,6 +119,7 @@ open class AuthService( companion object { private const val GUEST_TTL_MS = 7 * 24 * 60 * 60 * 1000L + private val passwordDispatcher = Dispatchers.Default.limitedParallelism(2, "password-hashing") fun fixed(userId: String): AuthService = object : AuthService("test") { override suspend fun verify(token: String): String? = if (token == "test-jwt") userId else null diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt index 08d69779..d2bb7f4c 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt @@ -1,8 +1,10 @@ package dev.typetype.server import dev.typetype.server.services.AuthService +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout @@ -13,8 +15,33 @@ import org.junit.jupiter.api.Test import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import kotlin.math.max class AuthServiceDispatcherTest { + @Test + fun `database work remains available when shared IO is saturated`() { + val ioParallelism = max(64, Runtime.getRuntime().availableProcessors()) + val entered = CountDownLatch(ioParallelism) + val release = CountDownLatch(1) + val service = AuthService("test-secret", hasUsersProbe = { true }) + + runBlocking { + val blockers = List(ioParallelism) { + async(Dispatchers.IO) { + entered.countDown() + release.await() + } + } + try { + assertTrue(entered.await(5, TimeUnit.SECONDS)) + assertTrue(withTimeout(1_000) { service.hasUsers() }) + } finally { + release.countDown() + blockers.awaitAll() + } + } + } + @Test fun `slow database work does not block the caller dispatcher`() { val entered = CountDownLatch(1) From 35776cccbaddf193e78106022bfba972fd128103 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 16:19:00 +0200 Subject: [PATCH 55/65] fix: warm live playback behind edge --- .../server/services/SabrSessionPump.kt | 15 ++++++-- .../services/SabrLiveSessionWarmupTest.kt | 38 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 5b0e12ac..11280335 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -22,7 +22,7 @@ internal class SabrSessionPump( var liveWarmupTarget: SabrLiveWarmupTarget? = null holder.setPlaybackState(SabrPlaybackState.PREPARING) while (pumps < maxPumps && - !isWarmEnough(holder) && + !isWarmEnough(holder, liveWarmupTarget) && (!holder.session.isComplete || holder.expectsLive()) ) { val currentLiveTarget = liveWarmupTarget @@ -56,10 +56,19 @@ internal class SabrSessionPump( holder.setPlaybackState(SabrPlaybackState.IDLE) } - private fun isWarmEnough(holder: SabrSessionHolder): Boolean { + private fun isWarmEnough(holder: SabrSessionHolder, liveTarget: SabrLiveWarmupTarget?): Boolean { val audioObserved = holder.observedMediaSegment(holder.audioFormat) != null val videoObserved = !holder.isVideoActive() || holder.observedMediaSegment(holder.videoFormat) != null - if (holder.expectsLive()) return audioObserved && videoObserved + if (holder.expectsLive()) { + val target = liveTarget ?: return false + val audioStartMs = holder.earliestObservedMediaStartMs(holder.audioFormat) ?: return false + val videoStartMs = if (holder.isVideoActive()) { + holder.earliestObservedMediaStartMs(holder.videoFormat) ?: return false + } else { + audioStartMs + } + return maxOf(audioStartMs, videoStartMs) <= target.timeMs + target.segmentDurationMs + } return bothFormatsKnown(holder) || holder.session.streamState.getMaxSegment(holder.audioFormat) > 0 && holder.session.streamState.getMaxSegment(holder.videoFormat) > 0 diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt index be3caec6..30e2aa12 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -17,6 +17,44 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveSessionWarmupTest { + @Test + fun `warmup continues when the first media pair is too close to the live head`() = runTest { + val audio = format(140, audio = true, "audio/mp4") + val video = format(299, audio = false, "video/mp4") + val streamState = mockk(relaxed = true) + val session = mockk() + var pumps = 0 + every { session.streamState } returns streamState + every { session.isComplete } returns false + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 1_000L + every { streamState.isLive } returns true + every { streamState.isPostLiveDvr } returns false + every { streamState.liveHeadSequenceNumber } returns 1_000L + every { streamState.liveHeadTimeMs } returns 2_000_000L + val audioInit = mp4Box("ftyp", byteArrayOf(1)) + mp4Box("moov", byteArrayOf(2)) + val videoInit = mp4Box("ftyp", byteArrayOf(3)) + mp4Box("moov", byteArrayOf(4)) + every { session.pumpOnce(any()) } answers { + pumps++ + val atTarget = pumps > 1 + val sequence = if (atTarget) 990 else 1_000 + val startMs = if (atTarget) 1_980_000L else 2_000_000L + listOf( + segment(140, sequence, startMs, 2_000L, audioInit + mediaFragment(5)), + segment(299, sequence, startMs, 2_000L, videoInit + mediaFragment(6)), + ) + } + val holder = holder(session, audio, video) + holder.markExpectedLive() + + SabrSessionPump(SabrSegmentCache()).ensureWarmed(holder, maxPumps = 8) + + assertEquals(2, pumps) + assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(audio)) + assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(video)) + assertEquals(1_980_000L, holder.resolvePlaybackStartMs(0L)) + } + @Test fun `warmup keeps bootstrap initialization and requests real live media`() = runTest { val audio = format(140, audio = true, "audio/mp4") From 37c44c927131afad59923c4da7e6147cffcde325 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 16:37:11 +0200 Subject: [PATCH 56/65] fix: preserve contiguous live playback ranges --- .../services/SabrLiveContinuationRequest.kt | 18 ++++++------- .../SabrLiveContinuationRequestTest.kt | 26 ++++++++++++++++--- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt index 6d6f8e0f..2b690d87 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt @@ -8,8 +8,8 @@ internal inline fun withLiveContinuationRequestShape( block: () -> T, ): T { val ranges = buildList { - holder.observedRange(holder.audioFormat)?.takeIf { holder.isAudioActive() }?.let(::add) - holder.observedRange(holder.videoFormat)?.takeIf { holder.isVideoActive() }?.let(::add) + holder.continuationRange(holder.audioFormat)?.takeIf { holder.isAudioActive() }?.let(::add) + holder.continuationRange(holder.videoFormat)?.takeIf { holder.isVideoActive() }?.let(::add) } if (ranges.isEmpty()) return block() val state = holder.session.streamState @@ -24,20 +24,18 @@ internal inline fun withLiveContinuationRequestShape( } } -private fun SabrSessionHolder.observedRange(format: YoutubeSabrFormat): SabrBufferedRange? { - val header = observedMediaSegment(format)?.header ?: return null - val sequence = header.sequenceNumber.takeIf { it > 0 } ?: return null - val startMs = header.startMs.takeIf { it >= 0L } ?: return null - val durationMs = header.durationMs.takeIf { it > 0L } - ?: playbackSegmentDurationMs(format, sequence) - val bufferedEndMs = startMs + durationMs +private fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { + observedMediaSegment(format) ?: return null + val sequence = lastServedSequence(format) + ?: (playbackStartSequence(format, requestedSeekTimeMs() ?: playerTimeMs()) - 1).coerceAtLeast(0) + val bufferedEndMs = playbackSegmentEndMs(format, sequence).coerceAtLeast(1L) return SabrBufferedRange( format.itag, format.lastModified, format.xtags, 0L, bufferedEndMs, - 1, + if (sequence > 0) 1 else 0, sequence, TIMESCALE, ) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt index 479d317f..aa9c30c9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt @@ -17,10 +17,12 @@ import java.time.Instant class SabrLiveContinuationRequestTest { @Test - fun `continuation advertises the exact observed live range`() { + fun `continuation advertises only media served to the player`() { val fixture = fixture() fixture.holder.observeMediaSegment(segment(fixture.audio.itag, 10_396, 10_395_000L, -1L)) fixture.holder.observeMediaSegment(segment(fixture.video.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.setLastServedSequence(fixture.audio.itag, 10_392) + fixture.holder.setLastServedSequence(fixture.video.itag, 10_392) fixture.holder.setPlayerTimeMs(10_390_500L) val result = withLiveContinuationRequestShape(fixture.holder) { "pumped" } @@ -28,8 +30,8 @@ class SabrLiveContinuationRequestTest { assertEquals("pumped", result) assertEquals( listOf( - "itag=140:seq=1-10396:time=0+10396000:timescale=1000", - "itag=299:seq=1-10396:time=0+10396000:timescale=1000", + "itag=140:seq=1-10392:time=0+10392000:timescale=1000", + "itag=299:seq=1-10392:time=0+10392000:timescale=1000", ), requireNotNull(fixture.rangeOverrides.first()).map(SabrBufferedRange::summarize), ) @@ -37,6 +39,24 @@ class SabrLiveContinuationRequestTest { verify { fixture.state.setPlayerTimeMs(10_390_500L) } } + @Test + fun `continuation does not advertise an unserved live edge`() { + val fixture = fixture() + fixture.holder.observeMediaSegment(segment(fixture.audio.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.observeMediaSegment(segment(fixture.video.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.setPlayerTimeMs(10_390_500L) + + withLiveContinuationRequestShape(fixture.holder) { Unit } + + assertEquals( + listOf( + "itag=140:seq=1-10390:time=0+10390000:timescale=1000", + "itag=299:seq=1-10390:time=0+10390000:timescale=1000", + ), + requireNotNull(fixture.rangeOverrides.first()).map(SabrBufferedRange::summarize), + ) + } + @Test fun `continuation leaves request state unchanged before media is observed`() { val fixture = fixture() From dcf9cde0c1ddac6b0f64a671190e1be1837a325e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 16:53:30 +0200 Subject: [PATCH 57/65] fix: recover live segments behind head --- .../server/services/SabrLivePlayback.kt | 6 ++- .../services/SabrLiveFutureRequestTest.kt | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index 4d21ed59..c2e3182f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -75,8 +75,12 @@ private fun SabrSessionHolder.availableLiveMediaStartMs(): Long? { internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false - livePlaybackSnapshot()?.takeIf { it.active } ?: return false + val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false if (session.getCachedSegment(request) != null) return false + if (request.format.itag == videoFormat.itag && + live.headSequence > 0L && + request.sequenceNumber.toLong() < live.headSequence + ) return false if (session.getReadableSegment(request) != null && !isHistoricalLiveRequest(request)) return true val state = session.streamState observedMediaSegment(request.format)?.let { observed -> diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt new file mode 100644 index 00000000..a18d527d --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrLiveFutureRequestTest { + @Test + fun `segment behind the reported live head is not treated as future`() { + val audio = format(140) + val video = format(299) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 230L + every { session.getCachedSegment(any()) } returns null + every { state.isLive } returns true + every { state.liveHeadSequenceNumber } returns 230L + every { state.liveHeadTimeMs } returns 1_060_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + assertFalse(holder.isFutureLiveRequest(SabrSegmentRequest.media(video, 201))) + } + + private fun format(itag: Int): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + } +} From 967aab38983ce75ea6be466c24668aa90592b9d5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 17:06:06 +0200 Subject: [PATCH 58/65] fix: recover lagging live audio segments --- .../server/services/SabrLivePlayback.kt | 4 ++ .../services/SabrLiveFutureRequestTest.kt | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index c2e3182f..d2b849fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -81,6 +81,10 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): live.headSequence > 0L && request.sequenceNumber.toLong() < live.headSequence ) return false + if (observedMediaSegment(request.format) != null && + playbackSegmentEndMs(request.format, request.sequenceNumber) < + live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS + ) return false if (session.getReadableSegment(request) != null && !isHistoricalLiveRequest(request)) return true val state = session.streamState observedMediaSegment(request.format)?.let { observed -> diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt index a18d527d..7d8b241b 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt @@ -4,6 +4,8 @@ import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo @@ -38,7 +40,45 @@ class SabrLiveFutureRequestTest { assertFalse(holder.isFutureLiveRequest(SabrSegmentRequest.media(video, 201))) } + @Test + fun `audio segment behind the live time is not treated as future`() { + val audio = format(140) + val video = format(299) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 3_920L + every { session.getCachedSegment(any()) } returns null + every { state.isLive } returns true + every { state.liveHeadSequenceNumber } returns 3_920L + every { state.liveHeadTimeMs } returns 7_842_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.observeMediaSegment(segment(audio.itag, 3_889, 7_776_000L)) + + assertFalse(holder.isFutureLiveRequest(SabrSegmentRequest.media(audio, 3_890))) + } + private fun format(itag: Int): YoutubeSabrFormat = mockk { every { this@mockk.itag } returns itag } + + private fun segment(itag: Int, sequence: Int, startMs: Long): SabrMediaSegment { + val header = mockk { + every { isInitSegment } returns false + every { this@mockk.itag } returns itag + every { sequenceNumber } returns sequence + every { this@mockk.startMs } returns startMs + every { durationMs } returns 2_000L + } + return mockk { every { this@mockk.header } returns header } + } } From 6ee01714aa444e481c4da1b8b17f5cbeab897ae1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 17:21:02 +0200 Subject: [PATCH 59/65] fix: recover contiguous live media gaps --- .../dev/typetype/server/services/SabrLivePlayback.kt | 8 ++++---- .../typetype/server/services/SabrLiveFutureRequestTest.kt | 6 +++++- .../server/services/SabrSeekRepositionPumpTest.kt | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index d2b849fa..f01c902e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -108,13 +108,13 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false + val observed = observedMediaSegment(request.format) ?: return false + val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) + if (requestEndMs < live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS) return true lastServedSequence(request.format)?.let { lastServed -> if (request.sequenceNumber in lastServed..lastServed + LIVE_FUTURE_SEGMENT_TOLERANCE) return false } - val observed = observedMediaSegment(request.format) ?: return false - if (request.sequenceNumber < observed.header.sequenceNumber) return true - val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) - return requestEndMs < live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS + return request.sequenceNumber < observed.header.sequenceNumber } internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt index 7d8b241b..9151234e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment @@ -63,8 +64,11 @@ class SabrLiveFutureRequestTest { lastRequestAt = Instant.EPOCH, ) holder.observeMediaSegment(segment(audio.itag, 3_889, 7_776_000L)) + holder.setLastServedSequence(audio.itag, 3_889) + val request = SabrSegmentRequest.media(audio, 3_890) - assertFalse(holder.isFutureLiveRequest(SabrSegmentRequest.media(audio, 3_890))) + assertFalse(holder.isFutureLiveRequest(request)) + assertTrue(holder.isHistoricalLiveRequest(request)) } private fun format(itag: Int): YoutubeSabrFormat = mockk { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index ca8ae740..f07ad191 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -196,6 +196,7 @@ class SabrSeekRepositionPumpTest { every { session.pumpOnceStreamingForDemand(any(), request) } returns mockk(relaxed = true) val holder = holder(session, audio, video) holder.observeMediaSegment(mediaSegment(video.itag, sequence = 3_076)) + holder.setLastServedSequence(video.itag, 3_076) holder.requestSegmentDemand(request) assertFalse(holder.isFutureLiveRequest(request)) assertEquals(DEFAULT_PLAYBACK_RETRY_MS, holder.liveRetryAfterMs(listOf(request))) From 949ec548f529f4cb9d47426ab4f420bb66fcac32 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 17:53:22 +0200 Subject: [PATCH 60/65] test: cover hidden live feed reconstruction --- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index 15a357b3..e4d94975 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -26,6 +26,7 @@ import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.mockk +import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue @@ -52,12 +53,27 @@ class SubscriptionFeedLiveVisibilityRoutesTest { duration = 0L, publishedAt = System.currentTimeMillis() + 86_400_000L, ), + video(3_250L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + ), video(3_000L, url = "https://youtube.com/watch?v=normal-1"), video(2_000L, url = "https://youtube.com/watch?v=normal-2"), ) feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) } + @Test + fun `disabled setting keeps every live state and normal videos`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + val videoIds = readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }.toSet() + assertEquals(setOf("live", "scheduled", "replay", "normal-1", "normal-2"), videoIds) + } + @Test fun `account setting hides live streams before pagination`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) @@ -129,6 +145,38 @@ class SubscriptionFeedLiveVisibilityRoutesTest { assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) } + @Test + fun `cached snapshot keeps live filtering after service reconstruction`() = runBlocking { + val cache = FakeCacheService() + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( + duration = 0L, + publishedAt = System.currentTimeMillis() + 86_400_000L, + ), + video(3_000L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + ), + video(2_000L, url = "https://youtube.com/watch?v=normal"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, cache) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + withApp { + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + feedService = SubscriptionFeedService(subscriptionsService, channelService, cache) + withApp { + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + } + @Test fun `account setting hides members only videos before pagination`() = withApp { val channelService = mockk() From b2ce4600f66b5e6495ead184adc196b3a7f64402 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 18:15:21 +0200 Subject: [PATCH 61/65] fix: recover transitioning live playback --- .../services/SabrPlaybackSessionService.kt | 33 ++++--- .../SabrTransitioningLivePlaybackTest.kt | 92 +++++++++++++++++++ 2 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index dadf06b0..b3fb8190 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -33,13 +33,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS ) if (isLive || prepared.isLive) holder.markExpectedLive() if (holder.expectsLive()) { - holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) - holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) - if (startTimeMs == 0L) { - holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) - holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) - } - sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) + prepareLive(holder, startTimeMs, audioOnly) } else { val initialization = SabrPlaybackInitializationPreloader.preload( sessionStore, @@ -48,11 +42,16 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS INITIALIZATION_PRELOAD_TIMEOUT_MS, ) if (!initialization.isComplete(audioOnly)) { - val missing = initialization.missingTracks(audioOnly, video.itag, audio.itag) - holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) - holder.setPlayerTimeMs(startTimeMs) - holder.failTerminal(sabrRecoverableFailureMessage("SABR initialization unavailable for $missing")) - return SabrPlaybackPreparation(holder, startTimeMs, ready = false) + if (holder.livePlaybackSnapshot()?.active == true) { + holder.markExpectedLive() + prepareLive(holder, startTimeMs, audioOnly) + } else { + val missing = initialization.missingTracks(audioOnly, video.itag, audio.itag) + holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) + holder.setPlayerTimeMs(startTimeMs) + holder.failTerminal(sabrRecoverableFailureMessage("SABR initialization unavailable for $missing")) + return SabrPlaybackPreparation(holder, startTimeMs, ready = false) + } } } return SabrPlaybackStarter.start( @@ -64,6 +63,16 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS ) } + private suspend fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { + holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) + holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) + if (startTimeMs == 0L) { + holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) + holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) + } + sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) + } + suspend fun seek( source: SabrSessionHolder, prepared: SabrPreparedInfo, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt new file mode 100644 index 00000000..0cdbfa61 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt @@ -0,0 +1,92 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrTransitioningLivePlaybackTest { + @Test + fun `live protocol response overrides stale ended metadata`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val info = mockk() + val prepared = SabrPreparedInfo(info, token(), isLive = false, isLiveContent = true) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 5_536L + every { state.isLive } returns true + every { state.isPostLiveDvr } returns false + every { state.liveHeadTimeMs } returns 11_070_200L + every { state.liveHeadSequenceNumber } returns 5_536L + val holder = SabrSessionHolder( + session = session, + info = info, + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + val store = mockk() + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 0L, + false, + SabrSessionPurpose.PLAYBACK, + false, + 0L, + ) + } returns holder + coEvery { store.fetchInitializationData(holder, video) } returns null + coEvery { store.fetchInitializationData(holder, audio) } returns null + coEvery { store.ensureWarmed(holder, 8) } returns Unit + every { store.startPump(holder) } returns Unit + + val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) + + assertTrue(holder.expectsLive()) + assertTrue(result.startTimeMs > 0L) + assertNull(holder.terminalFailure()) + assertEquals(11_050_200L, result.startTimeMs) + coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } + verify(exactly = 1) { store.startPump(holder) } + } + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { this@mockk.isAudio } returns isAudio + every { audioTrackId } returns null + every { mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { bitrate } returns if (isAudio) 128_000 else 5_000_000 + } + + private fun token(): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "visitor-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "visitor-data", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + ) +} From 18b2c3396e74b282eb0fda4955afd68a6007221a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 21 Aug 2026 18:35:31 +0200 Subject: [PATCH 62/65] fix: reuse authenticated SABR preparation --- .../services/AuthenticatedSabrInfoCache.kt | 45 +++++++++++++++++++ .../services/AuthenticatedSabrInfoService.kt | 10 ++++- .../AuthenticatedSabrInfoServiceTest.kt | 25 +++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt new file mode 100644 index 00000000..063e63ce --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CompletableDeferred +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap + +internal class AuthenticatedSabrInfoCache( + ttl: Duration = Duration.ofMinutes(5), + maxEntries: Int = 256, +) { + private val items = BoundedExpiringCache( + maxEntries = maxEntries, + ttl = ttl, + ) + private val inFlight = ConcurrentHashMap>() + + suspend fun getOrLoad( + credentials: YoutubeSessionCredentials, + videoId: String, + loader: suspend () -> AuthenticatedSabrInfoResult, + ): AuthenticatedSabrInfoResult { + val key = Key(credentials.userId, credentials.fingerprint, videoId) + items.get(key)?.let { return AuthenticatedSabrInfoResult.Ready(it) } + val pending = CompletableDeferred() + val existing = inFlight.putIfAbsent(key, pending) + if (existing != null) return existing.await() + return try { + val result = loader() + if (result is AuthenticatedSabrInfoResult.Ready) items.put(key, result.prepared) + pending.complete(result) + result + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + inFlight.remove(key, pending) + } + } + + private data class Key( + val userId: String, + val credentialFingerprint: String, + val videoId: String, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index 9e0dee20..b31ddb9f 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -16,11 +16,19 @@ internal class AuthenticatedSabrInfoService( private val tokenClient: TypetypeTokenSabrTokenClient, private val visitorDataFetcher: () -> String = AuthenticatedYoutubeVisitorData::fetch, private val probe: AuthenticatedSabrProbe = PipePipeAuthenticatedSabrProbe, + private val cache: AuthenticatedSabrInfoCache = AuthenticatedSabrInfoCache(), ) { suspend fun fetch(userId: String?, videoId: String): AuthenticatedSabrInfoResult { if (userId == null || userId.startsWith("guest:")) return AuthenticatedSabrInfoResult.NotConnected val credentials = youtubeSessionService.connectedCredentials(userId) ?: return AuthenticatedSabrInfoResult.NotConnected + return cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + } + + private suspend fun fetchUncached( + credentials: YoutubeSessionCredentials, + videoId: String, + ): AuthenticatedSabrInfoResult { return try { val prepared = YoutubeSessionTokenScope.withCredentials(credentials) { withContext(Dispatchers.IO) { @@ -36,7 +44,7 @@ internal class AuthenticatedSabrInfoService( ?: error("Authenticated SABR response has no audio and video formats") } } - youtubeSessionService.markUsed(userId) + youtubeSessionService.markUsed(credentials.userId) AuthenticatedSabrInfoResult.Ready(prepared) } catch (error: CancellationException) { throw error diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt index 84d72c34..9b6e3449 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -47,6 +47,31 @@ class AuthenticatedSabrInfoServiceTest { coVerify(exactly = 1) { sessions.markUsed(USER_ID) } } + @Test + fun `reuses authenticated info for the following playback request`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns sessionToken() + every { probe.fetch(VIDEO_ID, any()) } returns playableInfo() + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + val metadata = service.fetch(USER_ID, VIDEO_ID) + val playback = service.fetch(USER_ID, VIDEO_ID) + + assertSame((metadata as AuthenticatedSabrInfoResult.Ready).prepared, (playback as AuthenticatedSabrInfoResult.Ready).prepared) + verify(exactly = 1) { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } + verify(exactly = 1) { probe.fetch(VIDEO_ID, any()) } + coVerify(exactly = 1) { sessions.markUsed(USER_ID) } + } + @Test fun `guest playback does not inspect connected credentials`() = runTest { val sessions = mockk() From 5a06bd902aa068f0860f2b16991f60bb165ff5ab Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 10:16:42 +0200 Subject: [PATCH 63/65] fix: serialize subscription group mutations --- .../services/SubscriptionGroupsService.kt | 4 ++ .../server/SubscriptionGroupsServiceTest.kt | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 2566c5ed..9529ad45 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -37,6 +37,7 @@ class SubscriptionGroupsService { val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName val normalizedName = normalizeUniqueName(name) return DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) if (nameExists(userId, normalizedName)) return@query SubscriptionGroupWriteResult.DuplicateName val id = UUID.randomUUID().toString() val now = System.currentTimeMillis() @@ -59,6 +60,7 @@ class SubscriptionGroupsService { val normalizedName = normalizeUniqueName(name) return try { DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val current = SubscriptionGroupsTable.selectAll().where { (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) }.singleOrNull() ?: return@query SubscriptionGroupWriteResult.NotFound @@ -86,6 +88,7 @@ class SubscriptionGroupsService { } suspend fun delete(userId: String, groupId: String): Boolean = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) if (!groupExists(userId, groupId)) return@query false SubscriptionGroupMembershipsTable.deleteWhere { (SubscriptionGroupMembershipsTable.groupId eq groupId) and @@ -122,6 +125,7 @@ class SubscriptionGroupsService { groupId: String, rawChannelUrl: String, ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) val deleted = SubscriptionGroupMembershipsTable.deleteWhere { diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index f0bcfaca..4233deae 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -164,6 +164,54 @@ class SubscriptionGroupsServiceTest { assertTrue(groups.getChannelUrls(userId, group.id).all { it in subscriptionUrls }) } + @Test + fun `group mutations share the account subscription lock`() = runTest { + val userId = "concurrent-group-user" + subscriptions.add(userId, subscription("one")) + val renamedGroup = groups.create(userId, "Rename").createdGroup() + val deletedGroup = groups.create(userId, "Delete").createdGroup() + val membershipGroup = groups.create(userId, "Membership").createdGroup() + groups.addSubscription(userId, membershipGroup.id, channel("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val creation = async(Dispatchers.IO) { groups.create(userId, "Created") } + val rename = async(Dispatchers.IO) { groups.rename(userId, renamedGroup.id, "Renamed") } + val deletion = async(Dispatchers.IO) { groups.delete(userId, deletedGroup.id) } + val removal = async(Dispatchers.IO) { + groups.removeSubscription(userId, membershipGroup.id, channel("one")) + } + val allWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + var waiting = false + while (!waiting && !(creation.isCompleted && rename.isCompleted && deletion.isCompleted && removal.isCompleted)) { + waiting = waitingSubscriptionLocks(userId) >= 4 + if (!waiting) yield() + } + waiting + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assertTrue(creation.await() is SubscriptionGroupWriteResult.Success) + assertTrue(rename.await() is SubscriptionGroupWriteResult.Success) + assertTrue(deletion.await()) + assertEquals(SubscriptionGroupMembershipResult.Success, removal.await()) + assertTrue(allWaited, "all group mutations must wait for the account-scoped lock") + } + @Test fun `replacement imports retain only memberships for subscriptions still present`() = runTest { val group = groups.create("user", "Group").createdGroup() From a2f7eefa772e3809c985885cf222e7b3ed95a9c6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 10:16:53 +0200 Subject: [PATCH 64/65] fix: refresh legacy grouped feed snapshots --- .../services/SubscriptionFeedService.kt | 5 ++++ .../services/SubscriptionFeedSnapshot.kt | 3 ++ .../server/SubscriptionGroupFeedRoutesTest.kt | 28 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 2bef39f8..7581eb66 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -68,6 +68,11 @@ class SubscriptionFeedService( else -> store.previous(userId)?.takeIf { it.generation == cursorState.generation } ?: return SubscriptionFeedPageResult.StaleGeneration } + if (selection != SubscriptionSelection.All && !snapshot.hasCompleteSourceAttribution()) { + if (cursorState != null) return SubscriptionFeedPageResult.StaleGeneration + scheduleRefresh(userId, requestId) + return SubscriptionFeedPageResult.Preparing(PREPARING_RETRY_AFTER_MS) + } val offset = cursorState?.offset ?: page * limit val selected = selections.resolve(userId, selection, cursorState?.selectionToken) ?: return SubscriptionFeedPageResult.StaleGeneration diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 2110167f..436d7948 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -80,6 +80,9 @@ internal data class SubscriptionFeedCursorState( val selectionToken: String?, ) +internal fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = + videos.all { sourceChannelUrls.containsKey(it.subscriptionFeedKey()) } + internal fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt index e4c88e8c..7872dbb6 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt @@ -1,10 +1,13 @@ package dev.typetype.server +import dev.typetype.server.cache.CacheJson import dev.typetype.server.models.SubscriptionFeedResponse import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.routes.subscriptionFeedRoutes import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionFeedCacheKeys import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionFeedSnapshot import dev.typetype.server.services.SubscriptionGroupMembershipResult import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService @@ -161,6 +164,31 @@ class SubscriptionGroupFeedRoutesTest { assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) } + @Test + fun `group feed refreshes snapshots without source attribution`() = withApp { + val sourceUrl = channel("one") + val video = SubscriptionFeedTestFixtures.video(1_000L, channel = "different-canonical-uploader") + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, sourceUrl) + cache.set( + SubscriptionFeedCacheKeys.feed(TEST_USER_ID), + CacheJson.encodeToString( + SubscriptionFeedSnapshot.serializer(), + SubscriptionFeedSnapshot(1L, System.currentTimeMillis(), stale = false, videos = listOf(video)), + ), + 60, + ) + val channelService = mockk() + coEvery { channelService.getChannel(sourceUrl, null) } returns SubscriptionFeedTestFixtures.channel(video) + feed = SubscriptionFeedService(subscriptions, channelService, cache) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + } + private suspend fun ApplicationTestBuilder.requestReadyFeed( limit: Int = 30, groupId: String? = null, From 43b288ba1232e069f0bd0f8e5f67ada8b86c3e0b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 22 Aug 2026 11:16:08 +0200 Subject: [PATCH 65/65] chore: prepare server 1.6.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3f348d8d..aa33b2fc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.5.0 +appVersion=1.6.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000