Skip to content

Commit 7060644

Browse files
committed
fix(scraper): improve handling of Cloudflare challenges and empty Blogger responses
- Introduced `errBloggerVideoUnavailable` to handle cases where Blogger returns an empty response, allowing for fast-fail on dead tokens. - Updated `parseBatchexecuteResponse` to distinguish between empty responses and parsing failures, improving error handling. - Enhanced `GetVideoURLForEpisodeEnhanced` to propagate errors for unavailable Blogger videos. - Refined `checkChallengeDocument` to prevent false positives on legitimate pages mentioning "cloudflare" in non-challenge contexts. - Added comprehensive tests to ensure correct behavior for both legitimate pages and challenge signals, addressing issue #166. - Implemented diagnostic logging for failed stream extraction attempts, aiding in debugging.
1 parent 6d67a20 commit 7060644

10 files changed

Lines changed: 1148 additions & 11 deletions

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ require (
2828
github.com/andybalholm/brotli v1.2.1 // indirect
2929
github.com/atotto/clipboard v0.1.4 // indirect
3030
github.com/catppuccin/go v0.3.0 // indirect
31-
github.com/charmbracelet/ultraviolet v0.0.0-20260422141423-a0f1f21775f7 // indirect
31+
github.com/charmbracelet/ultraviolet v0.0.0-20260428153724-66037269d7be // indirect
3232
github.com/charmbracelet/x/exp/ordered v0.1.0 // indirect
3333
github.com/charmbracelet/x/exp/strings v0.1.0 // indirect
3434
github.com/charmbracelet/x/termios v0.1.1 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex
3030
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
3131
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
3232
github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
33-
github.com/charmbracelet/ultraviolet v0.0.0-20260422141423-a0f1f21775f7 h1:PeRlqWGEoO0apcS62iEgxQhVnFCTOYyQvi2sUTdf6IE=
34-
github.com/charmbracelet/ultraviolet v0.0.0-20260422141423-a0f1f21775f7/go.mod h1:3YdTxlnV/L0bQ3VN8WOSw8doF7LZV/xawUQ4MuAPDvo=
33+
github.com/charmbracelet/ultraviolet v0.0.0-20260428153724-66037269d7be h1:j7w8VP/D4lu5+/4GamMmFy8nrtadcl82/fjvDgSHwLo=
34+
github.com/charmbracelet/ultraviolet v0.0.0-20260428153724-66037269d7be/go.mod h1:3YdTxlnV/L0bQ3VN8WOSw8doF7LZV/xawUQ4MuAPDvo=
3535
github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI=
3636
github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ=
3737
github.com/charmbracelet/x/conpty v0.1.1 h1:s1bUxjoi7EpqiXysVtC+a8RrvPPNcNvAjfi4jxsAuEs=
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package player
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"strings"
7+
"testing"
8+
)
9+
10+
// Regression for the issue surfaced 2026-04-28 alongside the issue #166 fix.
11+
//
12+
// Symptom in the wild: a Goyabu episode whose Blogger token was dead upstream
13+
// (video deleted/region-blocked) caused 6 doomed batchexecute calls (3 in the
14+
// scraper retry loop + 3 in the player-layer redundant resolve), wasting
15+
// ~12 seconds before the user saw a generic "no valid video URL found".
16+
// Google signals "this RPC produced no result" with HTTP 200 + a body that is
17+
// only the `)]}'` anti-hijacking prefix.
18+
//
19+
// parseBatchexecuteResponse must distinguish that empty-body case
20+
// (errBloggerVideoUnavailable, which callers fast-fail on) from a structured
21+
// response that we just failed to walk (a generic error, which is worth
22+
// retrying because Google's response shape has shifted before — see the
23+
// 2026-04-23 fix in the function's docstring).
24+
func TestParseBatchexecuteResponse_EmptyBodyReturnsUnavailableSentinel(t *testing.T) {
25+
t.Parallel()
26+
27+
cases := []struct {
28+
name string
29+
body string
30+
}{
31+
{
32+
name: "bare anti-hijacking prefix",
33+
body: `)]}'` + "\n",
34+
},
35+
{
36+
name: "prefix with surrounding whitespace",
37+
body: "\n )]}'\n\n",
38+
},
39+
{
40+
name: "prefix only no newline",
41+
body: `)]}'`,
42+
},
43+
}
44+
45+
for _, tc := range cases {
46+
tc := tc
47+
t.Run(tc.name, func(t *testing.T) {
48+
t.Parallel()
49+
50+
_, err := parseBatchexecuteResponse([]byte(tc.body))
51+
if err == nil {
52+
t.Fatalf("expected error for empty batchexecute body, got nil")
53+
}
54+
if !errors.Is(err, errBloggerVideoUnavailable) {
55+
t.Fatalf("expected errBloggerVideoUnavailable, got %v", err)
56+
}
57+
})
58+
}
59+
}
60+
61+
// A non-empty response with no recognizable streams is *not* the
62+
// "video unavailable" case — it's a parser-shape problem worth retrying.
63+
// This guards the sentinel from being over-broad.
64+
func TestParseBatchexecuteResponse_StructuredButUnparsedIsNotUnavailable(t *testing.T) {
65+
t.Parallel()
66+
67+
// Valid wrb.fr envelope but the inner data has no streams array.
68+
body := `)]}'
69+
70+
41
71+
[["wrb.fr","WcwnYd","[\"some-payload\",null,1]",null,null,null,"generic"]]
72+
`
73+
74+
_, err := parseBatchexecuteResponse([]byte(body))
75+
if err == nil {
76+
t.Fatalf("expected error, got nil")
77+
}
78+
if errors.Is(err, errBloggerVideoUnavailable) {
79+
t.Fatalf("structured-but-unparsed must NOT be classified as unavailable, got %v", err)
80+
}
81+
if !strings.Contains(err.Error(), "no video URL found") {
82+
t.Fatalf("expected generic parse error, got %v", err)
83+
}
84+
}
85+
86+
// Sanity: a real-shape response with a googlevideo.com URL still parses
87+
// successfully. This protects against the empty-body branch swallowing the
88+
// happy path.
89+
func TestParseBatchexecuteResponse_RegexFallbackPicksGoogleVideoURL(t *testing.T) {
90+
t.Parallel()
91+
92+
body := `)]}'
93+
94+
128
95+
[["wrb.fr","WcwnYd","[null,null,[[[\"https://rr1---sn-xxx.googlevideo.com/videoplayback?expire=123&mime=video%2Fmp4&itag=22\"]]]]",null,null,null,"generic"]]
96+
`
97+
98+
got, err := parseBatchexecuteResponse([]byte(body))
99+
if err != nil {
100+
t.Fatalf("expected success, got %v", err)
101+
}
102+
if !strings.Contains(got, "googlevideo.com") {
103+
t.Fatalf("expected googlevideo URL, got %q", got)
104+
}
105+
}
106+
107+
// TestParseBatchexecuteResponse_TrulyEmptyBody covers a zero-byte body —
108+
// the most degenerate case. Real Blogger responses always include the
109+
// `)]}'` prefix, but a connection that closes mid-stream can produce
110+
// this. Treat it as token-unavailable so the retry loop fast-fails just
111+
// as it does for the prefix-only case.
112+
func TestParseBatchexecuteResponse_TrulyEmptyBody(t *testing.T) {
113+
t.Parallel()
114+
115+
_, err := parseBatchexecuteResponse(nil)
116+
if err == nil {
117+
t.Fatalf("expected error for nil body, got nil")
118+
}
119+
if !errors.Is(err, errBloggerVideoUnavailable) {
120+
t.Fatalf("nil body must map to errBloggerVideoUnavailable, got %v", err)
121+
}
122+
123+
_, err = parseBatchexecuteResponse([]byte(""))
124+
if err == nil {
125+
t.Fatalf("expected error for empty body, got nil")
126+
}
127+
if !errors.Is(err, errBloggerVideoUnavailable) {
128+
t.Fatalf("empty body must map to errBloggerVideoUnavailable, got %v", err)
129+
}
130+
131+
_, err = parseBatchexecuteResponse([]byte(" \n\t\n"))
132+
if err == nil {
133+
t.Fatalf("expected error for whitespace-only body, got nil")
134+
}
135+
if !errors.Is(err, errBloggerVideoUnavailable) {
136+
t.Fatalf("whitespace-only body must map to errBloggerVideoUnavailable, got %v", err)
137+
}
138+
}
139+
140+
// TestParseBatchexecuteResponse_SentinelSurvivesWrap proves the sentinel
141+
// remains identifiable through fmt.Errorf("…: %w", err). This is the
142+
// exact wrap used by extractActualVideoURL → GetVideoURLForEpisodeEnhanced
143+
// when surfacing the error to the caller, and it is what the retry-loop
144+
// fast-fail and the player-layer skip rely on. If anyone replaces %w
145+
// with %v in the wrap, this test fires.
146+
func TestParseBatchexecuteResponse_SentinelSurvivesWrap(t *testing.T) {
147+
t.Parallel()
148+
149+
_, raw := parseBatchexecuteResponse([]byte(`)]}'` + "\n"))
150+
if raw == nil {
151+
t.Fatalf("expected raw error from empty-body parse, got nil")
152+
}
153+
154+
wrapped := fmt.Errorf("video unavailable on this source: %w",
155+
fmt.Errorf("failed to extract video URL: %w", raw))
156+
157+
if !errors.Is(wrapped, errBloggerVideoUnavailable) {
158+
t.Fatalf("sentinel must survive double-wrap; got %v", wrapped)
159+
}
160+
}
161+
162+
// TestParseBatchexecuteResponse_LiveLogReplay locks down the production
163+
// payload shape observed on 2026-04-28 16:12:30 for the working Black
164+
// Clover episode 8 — data[2] streams, googlevideo CDN URL, mp4 mime.
165+
// Built via the same buildBatchexecuteBody helper used by the rest of
166+
// the suite so the JSON shape is canonical. If Google later changes
167+
// the wire format and we silently regress, this snapshot fires.
168+
func TestParseBatchexecuteResponse_LiveLogReplay(t *testing.T) {
169+
t.Parallel()
170+
171+
body := buildBatchexecuteBody(2, []string{
172+
"https://rr5---sn-q4f7l.googlevideo.com/videoplayback?expire=1745958900&itag=18&mime=video%2Fmp4",
173+
"https://rr5---sn-q4f7l.googlevideo.com/videoplayback?expire=1745958900&itag=22&mime=video%2Fmp4",
174+
})
175+
176+
got, err := parseBatchexecuteResponse(body)
177+
if err != nil {
178+
t.Fatalf("expected success on the production-shape payload, got %v", err)
179+
}
180+
if !strings.Contains(got, "googlevideo.com") {
181+
t.Fatalf("URL must be a googlevideo CDN URL, got %q", got)
182+
}
183+
if !strings.Contains(got, "itag=22") {
184+
t.Fatalf("expected 720p (itag=22) preference, got %q", got)
185+
}
186+
}
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Package player — Contract tests for the dead-Blogger-token fast-fail.
2+
//
3+
// 2026-04-28: a Goyabu episode whose Blogger token was dead upstream
4+
// (video deleted/region-blocked) caused 6 doomed batchexecute attempts
5+
// (3 in extractBloggerVideoURL's retry loop + 3 in the player layer's
6+
// HandleDownloadAndPlay redundant resolve), wasting ~12 seconds before
7+
// the user saw a generic "no valid video URL found".
8+
//
9+
// The fix introduces errBloggerVideoUnavailable as a sentinel that
10+
// propagates through:
11+
//
12+
// parseBatchexecuteResponse ──► extractBloggerGoogleVideoURL
13+
// ──► startBloggerProxy ("failed to extract video URL: %w")
14+
// ──► extractBloggerVideoURL (retry-loop fast-fail check)
15+
// ──► extractActualVideoURL
16+
// ──► GetVideoURLForEpisodeEnhanced ("video unavailable on this source: %w")
17+
// ──► PlayEpisode (routes user back to episode selection)
18+
//
19+
// Each link in this chain must preserve the sentinel via errors.Is.
20+
// These tests lock down the contract: anyone replacing %w with %v, or
21+
// re-wrapping with errors.New, will see this fire.
22+
package player
23+
24+
import (
25+
"errors"
26+
"fmt"
27+
"strings"
28+
"testing"
29+
)
30+
31+
// TestSentinelPropagation_ThroughProductionWrapChain reproduces the
32+
// exact wrap sequence used at every intermediate call site in the
33+
// dead-Blogger-token path and asserts errors.Is still finds the
34+
// sentinel at the top.
35+
func TestSentinelPropagation_ThroughProductionWrapChain(t *testing.T) {
36+
t.Parallel()
37+
38+
// Step 1 — parser detects empty body and returns the sentinel.
39+
_, leaf := parseBatchexecuteResponse([]byte(`)]}'` + "\n"))
40+
if leaf == nil {
41+
t.Fatalf("parser must return an error for empty body")
42+
}
43+
if !errors.Is(leaf, errBloggerVideoUnavailable) {
44+
t.Fatalf("parser must return errBloggerVideoUnavailable; got %v", leaf)
45+
}
46+
47+
// Step 2 — startBloggerProxy wraps with "failed to extract video URL: %w".
48+
step2 := fmt.Errorf("failed to extract video URL: %w", leaf)
49+
if !errors.Is(step2, errBloggerVideoUnavailable) {
50+
t.Fatalf("startBloggerProxy wrap must preserve sentinel; got %v", step2)
51+
}
52+
53+
// Step 3 — extractBloggerVideoURL's retry loop must observe the
54+
// sentinel so it can short-circuit. Verify a downstream caller
55+
// receiving the wrapped error can still detect it.
56+
if !errors.Is(step2, errBloggerVideoUnavailable) {
57+
t.Fatalf("retry-loop check must succeed on wrapped error")
58+
}
59+
60+
// Step 4 — GetVideoURLForEpisodeEnhanced wraps with
61+
// "video unavailable on this source: %w".
62+
step4 := fmt.Errorf("video unavailable on this source: %w", step2)
63+
if !errors.Is(step4, errBloggerVideoUnavailable) {
64+
t.Fatalf("GetVideoURLForEpisodeEnhanced wrap must preserve sentinel; got %v", step4)
65+
}
66+
67+
// Final — the message at the top of the chain must remain user-facing.
68+
if !strings.Contains(step4.Error(), "video unavailable on this source") {
69+
t.Fatalf("top-level error message must include user-facing prefix; got %q", step4.Error())
70+
}
71+
if !strings.Contains(step4.Error(), "failed to extract video URL") {
72+
t.Fatalf("top-level error chain must preserve intermediate context; got %q", step4.Error())
73+
}
74+
}
75+
76+
// TestSentinelPropagation_ProductionWrapMustUseW guards against a
77+
// subtle regression: someone replacing %w with %v in the wrap chain
78+
// (which kills errors.Is propagation). Mirrors the *exact* format
79+
// strings used in scraper.go so a wrong format string fails this test.
80+
func TestSentinelPropagation_ProductionWrapMustUseW(t *testing.T) {
81+
t.Parallel()
82+
83+
leaf := errBloggerVideoUnavailable
84+
85+
// The two wrap call sites in scraper.go.
86+
wrapA := fmt.Errorf("failed to extract video URL: %w", leaf)
87+
wrapB := fmt.Errorf("video unavailable on this source: %w", wrapA)
88+
89+
if !errors.Is(wrapA, errBloggerVideoUnavailable) {
90+
t.Fatalf("wrapA must preserve sentinel via %%w")
91+
}
92+
if !errors.Is(wrapB, errBloggerVideoUnavailable) {
93+
t.Fatalf("wrapB must preserve sentinel via %%w (caller depends on this)")
94+
}
95+
96+
// And the negative — using %v breaks errors.Is.
97+
bad := fmt.Errorf("failed to extract video URL: %v", leaf)
98+
if errors.Is(bad, errBloggerVideoUnavailable) {
99+
t.Fatalf("guard tripped: %%v should NOT preserve errors.Is — if it does, the Go runtime semantics changed and this whole assumption needs a re-audit")
100+
}
101+
}
102+
103+
// TestSentinelDistinctFromGenericParseError asserts that a structured
104+
// response we fail to walk (a different bug class from "video is dead")
105+
// does NOT match the sentinel. If this test breaks, it means the
106+
// sentinel got over-broad and would cause callers to fast-fail a
107+
// transient parser-shape problem that's worth retrying.
108+
func TestSentinelDistinctFromGenericParseError(t *testing.T) {
109+
t.Parallel()
110+
111+
// Valid wrb.fr envelope, but the inner data has no streams array — a
112+
// parser-shape problem worth retrying (the 2026-04-23 fix exists
113+
// precisely because Google reshuffled the index before).
114+
body := `)]}'
115+
116+
41
117+
[["wrb.fr","WcwnYd","[\"some-payload\",null,1]",null,null,null,"generic"]]
118+
`
119+
_, err := parseBatchexecuteResponse([]byte(body))
120+
if err == nil {
121+
t.Fatalf("expected error, got nil")
122+
}
123+
if errors.Is(err, errBloggerVideoUnavailable) {
124+
t.Fatalf("structured-but-unparsed must NOT match the dead-token sentinel; got %v", err)
125+
}
126+
}
127+
128+
// TestNeedsVideoExtraction_BloggerVariants is a guardrail for the URL
129+
// classifier that decides which URLs flow into extractBloggerVideoURL
130+
// (and therefore into the parser's sentinel path). If a future code
131+
// change loses Blogger URL detection, the dead-token fast-fail would
132+
// silently stop firing for that class. Pin every variant we serve.
133+
func TestNeedsVideoExtraction_BloggerVariants(t *testing.T) {
134+
t.Parallel()
135+
136+
cases := []struct {
137+
url string
138+
want bool
139+
why string
140+
}{
141+
{"https://www.blogger.com/video.g?token=XYZ", true, "canonical blogger embed"},
142+
{"https://blogger.com/video.g?token=XYZ", true, "blogger without www"},
143+
{"http://www.blogger.com/video.g?token=XYZ", true, "http (no scheme upgrade)"},
144+
{"https://www.blogspot.com/video/XYZ", true, "blogspot variant"},
145+
{"https://WWW.BLOGGER.COM/VIDEO.G?TOKEN=XYZ", true, "uppercase host (case-insensitive)"},
146+
{"https://rr1---sn-xxx.googlevideo.com/videoplayback?expire=123", false, "resolved CDN URL"},
147+
{"http://127.0.0.1:54577/blogger_proxy", false, "local proxy URL"},
148+
{"https://cdn.example.com/master.m3u8", false, "HLS"},
149+
{"", false, "empty"},
150+
}
151+
152+
for _, tc := range cases {
153+
tc := tc
154+
t.Run(tc.why, func(t *testing.T) {
155+
t.Parallel()
156+
157+
got := needsVideoExtraction(tc.url)
158+
if got != tc.want {
159+
t.Fatalf("needsVideoExtraction(%q) = %v, want %v (%s)", tc.url, got, tc.want, tc.why)
160+
}
161+
})
162+
}
163+
}

internal/player/goyabu_blogger_fix_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,9 +289,11 @@ func TestParseBatchexecuteResponse_FallbackRegex_StreamsSemMIME(t *testing.T) {
289289
func TestParseBatchexecuteResponse_CorpoVazio(t *testing.T) {
290290
t.Parallel()
291291

292+
// 2026-04-28: an empty body now maps to errBloggerVideoUnavailable so
293+
// callers can fast-fail dead Blogger tokens. See batchexecute_parse_test.go.
292294
_, err := parseBatchexecuteResponse([]byte{})
293295
assert.Error(t, err)
294-
assert.Contains(t, err.Error(), "no video URL found")
296+
assert.ErrorIs(t, err, errBloggerVideoUnavailable)
295297
}
296298

297299
func TestParseBatchexecuteResponse_JSONInvalido(t *testing.T) {

0 commit comments

Comments
 (0)