From 03395af8a1535245ef247f050951c2eb655b03e5 Mon Sep 17 00:00:00 2001 From: freya02 <41875020+freya022@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:51:12 +0200 Subject: [PATCH 1/4] Add interceptor module --- MediaThor-Interceptor/pom.xml | 33 ++++++++++ .../mediathor/interceptor/InterceptorMain.kt | 61 +++++++++++++++++++ pom.xml | 6 ++ 3 files changed, 100 insertions(+) create mode 100644 MediaThor-Interceptor/pom.xml create mode 100644 MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt diff --git a/MediaThor-Interceptor/pom.xml b/MediaThor-Interceptor/pom.xml new file mode 100644 index 0000000..b316778 --- /dev/null +++ b/MediaThor-Interceptor/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + io.github.freya022 + mediathor + 1.0 + + + mediathor-interceptor + MediaThor-Interceptor + + + io.github.freya022.mediathor.interceptor.InterceptorMain + + + + + com.google.code.gson + gson + + + com.squareup.okhttp3 + okhttp + + + org.seleniumhq.selenium + selenium-java + + + \ No newline at end of file diff --git a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt new file mode 100644 index 0000000..e415263 --- /dev/null +++ b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt @@ -0,0 +1,61 @@ +package io.github.freya022.mediathor.interceptor + +import mu.two.KotlinLogging +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.openqa.selenium.Platform +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.devtools.HasDevTools +import org.openqa.selenium.devtools.v116.network.Network +import org.openqa.selenium.edge.EdgeDriver +import org.openqa.selenium.remote.RemoteWebDriver +import java.io.IOException +import java.util.* +import kotlin.io.path.Path +import kotlin.io.path.absolutePathString +import kotlin.io.path.bufferedWriter +import kotlin.io.path.exists + +private val logger = KotlinLogging.logger { } + +object InterceptorMain { + private val outputPath = Path("master_playlists.txt") + + init { + if (outputPath.exists()) + throw IOException("${outputPath.absolutePathString()} already exists") + } + + private val writer = outputPath.bufferedWriter() + + @JvmStatic + fun main(args: Array) { + val driver = when (Platform.getCurrent().family()) { + Platform.WINDOWS -> EdgeDriver() + else -> ChromeDriver() + } + + Runtime.getRuntime().addShutdownHook(Thread { + writer.close() + driver.quit() + }) + + run(driver) + } + + private fun run(driver: T) where T : HasDevTools, T : RemoteWebDriver { + val devTools = driver.devTools + devTools.createSession() + devTools.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty())) + + devTools.addListener(Network.requestWillBeSent()) { futureRequest -> + val url = futureRequest.request.url + if (!url.toHttpUrl().pathSegments.last().endsWith(".m3u8")) { + logger.trace { "Discarding $url as it is not a '*.m3u8' url" } + return@addListener + } + + logger.info { "Added $url" } + writer.appendLine(url) + } + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index e5f9759..6534206 100644 --- a/pom.xml +++ b/pom.xml @@ -14,6 +14,7 @@ MediaThor-Volume MediaThor-Record-Helper MediaThor-Commons + MediaThor-Interceptor MediaThor @@ -210,6 +211,11 @@ jnr-winfsp c7be8901e8 + + org.seleniumhq.selenium + selenium-java + 4.12.0 + From 6163c6d2a9b48c6d79cea827c2262cff437ecc06 Mon Sep 17 00:00:00 2001 From: freya02 <41875020+freya022@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:19:13 +0200 Subject: [PATCH 2/4] Move interceptor to class, add overwrite switch --- .../src/main/resources/logback.xml | 6 +- MediaThor-Interceptor/.gitignore | 1 + MediaThor-Interceptor/pom.xml | 47 ++++++++++++++++ .../mediathor/interceptor/Interceptor.kt | 56 +++++++++++++++++++ .../mediathor/interceptor/InterceptorMain.kt | 52 ++--------------- 5 files changed, 113 insertions(+), 49 deletions(-) create mode 100644 MediaThor-Interceptor/.gitignore create mode 100644 MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt diff --git a/MediaThor-Commons/src/main/resources/logback.xml b/MediaThor-Commons/src/main/resources/logback.xml index 226c09a..4d13f53 100644 --- a/MediaThor-Commons/src/main/resources/logback.xml +++ b/MediaThor-Commons/src/main/resources/logback.xml @@ -6,11 +6,15 @@ - + + + + + diff --git a/MediaThor-Interceptor/.gitignore b/MediaThor-Interceptor/.gitignore new file mode 100644 index 0000000..916af1f --- /dev/null +++ b/MediaThor-Interceptor/.gitignore @@ -0,0 +1 @@ +/master_playlists.txt \ No newline at end of file diff --git a/MediaThor-Interceptor/pom.xml b/MediaThor-Interceptor/pom.xml index b316778..ba343b6 100644 --- a/MediaThor-Interceptor/pom.xml +++ b/MediaThor-Interceptor/pom.xml @@ -16,7 +16,32 @@ io.github.freya022.mediathor.interceptor.InterceptorMain + + + + org.apache.maven.plugins + maven-shade-plugin + + + + ${mainClass} + + + + false + ${project.name} + + + + + + + io.github.freya022 + mediathor-commons + 1.0 + com.google.code.gson gson @@ -28,6 +53,28 @@ org.seleniumhq.selenium selenium-java + + + org.seleniumhq.selenium + selenium-devtools-v85 + + + org.seleniumhq.selenium + selenium-devtools-v114 + + + org.seleniumhq.selenium + selenium-devtools-v115 + + + org.seleniumhq.selenium + selenium-firefox-driver + + + org.seleniumhq.selenium + selenium-ie-driver + + \ No newline at end of file diff --git a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt new file mode 100644 index 0000000..20f1565 --- /dev/null +++ b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt @@ -0,0 +1,56 @@ +package io.github.freya022.mediathor.interceptor + +import mu.two.KotlinLogging +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.openqa.selenium.Platform +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.devtools.v116.network.Network +import org.openqa.selenium.edge.EdgeDriver +import java.nio.file.StandardOpenOption +import java.util.* +import java.util.logging.Level +import kotlin.io.path.Path +import kotlin.io.path.bufferedWriter + +private val logger = KotlinLogging.logger { } + +class Interceptor { + private val writer = outputPath.bufferedWriter(options = arrayOf(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) + + init { + val driver = when (Platform.getCurrent().family()) { + Platform.WINDOWS -> EdgeDriver() + else -> ChromeDriver() + } + driver.setLogLevel(Level.INFO) + + Runtime.getRuntime().addShutdownHook(Thread { + writer.close() + driver.quit() + }) + + val devTools = driver.devTools + devTools.createSession() + devTools.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty())) + + devTools.addListener(Network.requestWillBeSent()) { futureRequest -> + val url = futureRequest.request.url + if (!url.startsWith("http")) { + logger.trace { "Discarding $url as it is not an http url" } + return@addListener + } + + if (!url.toHttpUrl().pathSegments.last().endsWith(".m3u8")) { + logger.trace { "Discarding $url as it is not a '*.m3u8' url" } + return@addListener + } + + logger.info { "Added $url" } + writer.appendLine(url) + } + } + + companion object { + val outputPath = Path("master_playlists.txt") + } +} \ No newline at end of file diff --git a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt index e415263..901d2d2 100644 --- a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt +++ b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt @@ -1,61 +1,17 @@ package io.github.freya022.mediathor.interceptor -import mu.two.KotlinLogging -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.openqa.selenium.Platform -import org.openqa.selenium.chrome.ChromeDriver -import org.openqa.selenium.devtools.HasDevTools -import org.openqa.selenium.devtools.v116.network.Network -import org.openqa.selenium.edge.EdgeDriver -import org.openqa.selenium.remote.RemoteWebDriver import java.io.IOException -import java.util.* -import kotlin.io.path.Path import kotlin.io.path.absolutePathString -import kotlin.io.path.bufferedWriter import kotlin.io.path.exists -private val logger = KotlinLogging.logger { } - object InterceptorMain { - private val outputPath = Path("master_playlists.txt") - - init { - if (outputPath.exists()) - throw IOException("${outputPath.absolutePathString()} already exists") - } - - private val writer = outputPath.bufferedWriter() - @JvmStatic fun main(args: Array) { - val driver = when (Platform.getCurrent().family()) { - Platform.WINDOWS -> EdgeDriver() - else -> ChromeDriver() - } - - Runtime.getRuntime().addShutdownHook(Thread { - writer.close() - driver.quit() - }) - - run(driver) - } - - private fun run(driver: T) where T : HasDevTools, T : RemoteWebDriver { - val devTools = driver.devTools - devTools.createSession() - devTools.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty())) + if (Interceptor.outputPath.exists() && "-y" !in args) + throw IOException("${Interceptor.outputPath.absolutePathString()} already exists") - devTools.addListener(Network.requestWillBeSent()) { futureRequest -> - val url = futureRequest.request.url - if (!url.toHttpUrl().pathSegments.last().endsWith(".m3u8")) { - logger.trace { "Discarding $url as it is not a '*.m3u8' url" } - return@addListener - } + Interceptor() - logger.info { "Added $url" } - writer.appendLine(url) - } + Thread.sleep(Long.MAX_VALUE) } } \ No newline at end of file From 5290f9e366f7b842ed0df176ccfdb35dc27f4ab6 Mon Sep 17 00:00:00 2001 From: freya02 <41875020+freya022@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:02:42 +0200 Subject: [PATCH 3/4] Try to improve the shutdown process --- .../mediathor/interceptor/Interceptor.kt | 58 +++++++++++++++---- .../mediathor/interceptor/InterceptorMain.kt | 7 ++- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt index 20f1565..ebe3f22 100644 --- a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt +++ b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt @@ -5,29 +5,41 @@ import okhttp3.HttpUrl.Companion.toHttpUrl import org.openqa.selenium.Platform import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.devtools.v116.network.Network +import org.openqa.selenium.devtools.v116.target.Target import org.openqa.selenium.edge.EdgeDriver +import org.openqa.selenium.remote.DriverCommand +import org.openqa.selenium.remote.RemoteWebDriver import java.nio.file.StandardOpenOption import java.util.* import java.util.logging.Level +import kotlin.coroutines.Continuation +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine import kotlin.io.path.Path import kotlin.io.path.bufferedWriter +import kotlin.jvm.optionals.getOrNull private val logger = KotlinLogging.logger { } -class Interceptor { +class Interceptor private constructor() { private val writer = outputPath.bufferedWriter(options = arrayOf(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) + private lateinit var continuation: Continuation + + private val driver = when (Platform.getCurrent().family()) { + Platform.WINDOWS -> EdgeDriver() + else -> ChromeDriver() + } + + private val hook = Thread { + driver.maybeGetDevTools().getOrNull()?.clearListeners() + driver.quit() + close() + } + init { - val driver = when (Platform.getCurrent().family()) { - Platform.WINDOWS -> EdgeDriver() - else -> ChromeDriver() - } driver.setLogLevel(Level.INFO) - - Runtime.getRuntime().addShutdownHook(Thread { - writer.close() - driver.quit() - }) + Runtime.getRuntime().addShutdownHook(hook) val devTools = driver.devTools devTools.createSession() @@ -48,9 +60,35 @@ class Interceptor { logger.info { "Added $url" } writer.appendLine(url) } + + devTools.addListener(Target.detachedFromTarget()) { + logger.info { "Exiting" } + Runtime.getRuntime().removeShutdownHook(hook) + continuation.resume(Unit) + } + } + + private suspend fun awaitTermination(): Unit = suspendCoroutine { + continuation = it + } + + // This method is only used when the browser is already closed. + private fun close() { + writer.close() } companion object { val outputPath = Path("master_playlists.txt") + + suspend fun intercept() { + val interceptor = Interceptor() + interceptor.awaitTermination() + // If the browser is closed, the driver is still running + // Using the QUIT command makes it exit + val executeMethod = RemoteWebDriver::class.java.getDeclaredMethod("execute", String::class.java) + executeMethod.isAccessible = true + executeMethod.invoke(interceptor.driver, DriverCommand.QUIT) + interceptor.close() + } } } \ No newline at end of file diff --git a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt index 901d2d2..f18b045 100644 --- a/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt +++ b/MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt @@ -1,5 +1,6 @@ package io.github.freya022.mediathor.interceptor +import kotlinx.coroutines.runBlocking import java.io.IOException import kotlin.io.path.absolutePathString import kotlin.io.path.exists @@ -10,8 +11,8 @@ object InterceptorMain { if (Interceptor.outputPath.exists() && "-y" !in args) throw IOException("${Interceptor.outputPath.absolutePathString()} already exists") - Interceptor() - - Thread.sleep(Long.MAX_VALUE) + runBlocking { + Interceptor.intercept() + } } } \ No newline at end of file From ee61909fa1aa4cde222113930009b9954f2067a6 Mon Sep 17 00:00:00 2001 From: freya02 <41875020+freya022@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:53:06 +0200 Subject: [PATCH 4/4] Add README.md --- MediaThor-Interceptor/README.md | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) create mode 100644 MediaThor-Interceptor/README.md diff --git a/MediaThor-Interceptor/README.md b/MediaThor-Interceptor/README.md new file mode 100644 index 0000000..0bdb32f --- /dev/null +++ b/MediaThor-Interceptor/README.md @@ -0,0 +1,9 @@ +# MediaThor-Interceptor + +A utility to intercept HLS stream requests and append them to a list + +## Limitations +* Master playlists won't be distinguished from media playlists + +## Requirements +* Chrome on Linux/Mac (Optional), or Chromium-based Edge on Windows \ No newline at end of file diff --git a/README.md b/README.md index ed5a2ef..40bc541 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,6 @@ These utilities are focused on specific use cases, the goal is NOT a swiss army ## Subprojects * [MediaThor-DL](MediaThor-DL): A downloader for HLS streams +* [MediaThor-Interceptor](MediaThor-Interceptor): Writes *.m3u8 files in a list from browser usage * [MediaThor-Volume](MediaThor-Volume): Helps you manage the volume of your music files * [MediaThor-Record-Helper](MediaThor-Record-Helper): Merges overlapping clips using a temporary ramdisk \ No newline at end of file