Skip to content

Commit a426c1a

Browse files
nikitachapovskii-devnikitachapovskii-dev
authored
feat: recommend specialized Actors via the TIP key (#147)
* feat: recommend specialized Actors via the TIP key * chore: don't crash on fail to store * chore: remove redundant Actor.fail * fix: detect listed sites in sentences and under country domains
1 parent 5038e66 commit a426c1a

6 files changed

Lines changed: 334 additions & 2 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
"joplin-turndown-plugin-gfm": "^1.0.12",
2020
"jsdom": "^24.1.1",
2121
"playwright": "1.55.1",
22+
"tldts": "^7.4.10",
2223
"turndown": "^7.2.0"
2324
},
2425
"devDependencies": {

‎pnpm-lock.yaml‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/const.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,6 @@ export enum ContentCrawlerTypes {
2121
export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60;
2222

2323
export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10;
24+
25+
/** Reserved key-value store key holding an advisory message about the run. */
26+
export const TIP_KVS_KEY = 'TIP';

‎src/main.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { getMiniActor } from './mini-actors.js';
77
import { addTimeoutToAllResponses } from './responses.js';
88
import { handleSearchNormalMode } from './search.js';
99
import { createServer } from './server.js';
10+
import { findActorTip, storeActorTip } from './tips.js';
1011
import type { Input } from './types.js';
1112
import { isActorStandby } from './utils.js';
1213

@@ -66,12 +67,20 @@ if (isActorStandby()) {
6667
contentScraperSettings ${JSON.stringify(contentScraperSettings)}
6768
`);
6869

70+
// Normal mode only: the key-value store of a standby run belongs to the Actor, not to the caller.
71+
const tip = findActorTip(input);
72+
if (tip) log.info(`Tip: ${tip.message}`);
73+
6974
let stats = { requestsFinished: 0, requestsFailed: 0 };
75+
let failure: Error | undefined;
7076
try {
7177
stats = await handleSearchNormalMode(input, searchCrawlerOptions, contentCrawlerOptions, contentScraperSettings);
7278
} catch (e) {
73-
const error = e as Error;
74-
await Actor.fail(error.message as string);
79+
failure = e as Error;
7580
}
81+
82+
await storeActorTip(tip);
83+
84+
if (failure) await Actor.fail(failure.message);
7685
await Actor.exit(`Finished! Scraped ${stats.requestsFinished} pages, ${stats.requestsFailed} failed.`);
7786
}

‎src/tips.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import { Actor } from 'apify';
2+
import { log } from 'crawlee';
3+
import { getDomainWithoutSuffix } from 'tldts';
4+
5+
import { TIP_KVS_KEY } from './const.js';
6+
7+
export type ActorTip = {
8+
message: string;
9+
level: 'info' | 'warning';
10+
};
11+
12+
type SiteRule = {
13+
site: string;
14+
actorTitle: string;
15+
actorUrl: string;
16+
domains: string[];
17+
matchesUrl?: (url: URL) => boolean;
18+
};
19+
20+
function hostnameMatchesDomain(hostname: string, domain: string): boolean {
21+
const normalizedHostname = hostname.toLowerCase().replace(/\.$/, '');
22+
23+
// `example.*` matches the site under any of its country domains, such as `amazon.co.uk`.
24+
if (domain.endsWith('.*')) {
25+
return getDomainWithoutSuffix(normalizedHostname) === domain.slice(0, -2);
26+
}
27+
28+
return normalizedHostname === domain || normalizedHostname.endsWith(`.${domain}`);
29+
}
30+
31+
function pathStartsWith(pathname: string, segment: string): boolean {
32+
return pathname === segment || pathname.startsWith(`${segment}/`);
33+
}
34+
35+
/** Specialized Actors we recommend instead of general-purpose scraping. The first matching rule wins. */
36+
const SITE_RULES: SiteRule[] = [
37+
{
38+
site: 'Facebook groups',
39+
actorTitle: 'Facebook Groups Scraper',
40+
actorUrl: 'https://apify.com/apify/facebook-groups-scraper',
41+
domains: ['facebook.com'],
42+
matchesUrl: (url) => pathStartsWith(url.pathname, '/groups'),
43+
},
44+
{
45+
site: 'facebook.com',
46+
actorTitle: 'Facebook Posts Scraper',
47+
actorUrl: 'https://apify.com/apify/facebook-posts-scraper',
48+
domains: ['facebook.com'],
49+
},
50+
{
51+
// Plain Google Search is what this Actor does itself, so only Google Maps gets a recommendation.
52+
site: 'Google Maps',
53+
actorTitle: 'Google Maps Scraper',
54+
actorUrl: 'https://apify.com/compass/crawler-google-places',
55+
domains: ['google.*'],
56+
matchesUrl: (url) => url.hostname.startsWith('maps.') || pathStartsWith(url.pathname, '/maps'),
57+
},
58+
{
59+
site: 'zillow.com',
60+
actorTitle: 'Zillow Detail Scraper',
61+
actorUrl: 'https://apify.com/maxcopell/zillow-detail-scraper',
62+
domains: ['zillow.com'],
63+
},
64+
{
65+
site: 'instagram.com',
66+
actorTitle: 'Instagram Scraper',
67+
actorUrl: 'https://apify.com/apify/instagram-scraper',
68+
domains: ['instagram.com'],
69+
},
70+
{
71+
site: 'booking.com',
72+
actorTitle: 'Booking Scraper',
73+
actorUrl: 'https://apify.com/voyager/booking-scraper',
74+
domains: ['booking.com'],
75+
},
76+
{
77+
site: 'tripadvisor.com',
78+
actorTitle: 'Tripadvisor Scraper',
79+
actorUrl: 'https://apify.com/maxcopell/tripadvisor',
80+
domains: ['tripadvisor.*'],
81+
},
82+
{
83+
site: 'youtube.com',
84+
actorTitle: 'YouTube Scraper',
85+
actorUrl: 'https://apify.com/streamers/youtube-scraper',
86+
domains: ['youtube.com', 'youtu.be'],
87+
},
88+
{
89+
site: 'tiktok.com',
90+
actorTitle: 'TikTok Scraper',
91+
actorUrl: 'https://apify.com/clockworks/tiktok-scraper',
92+
domains: ['tiktok.com'],
93+
},
94+
{
95+
site: 'amazon.com',
96+
actorTitle: 'Amazon Crawler',
97+
actorUrl: 'https://apify.com/junglee/amazon-crawler',
98+
domains: ['amazon.*'],
99+
},
100+
];
101+
102+
/**
103+
* Without this, whether a domain is recognized in a sentence depends on whether the punctuation after it
104+
* happens to be a URL delimiter: `zillow.com.` and `zillow.com?` parse fine, `zillow.com,` does not.
105+
*/
106+
const SURROUNDING_PUNCTUATION = /^["'([]+|["')\],.;:!?]+$/g;
107+
const SEARCH_OPERATOR_PREFIX = /^[a-z]+:(?!\/\/)/i;
108+
109+
function interpretTokenAsUrl(token: string): URL | null {
110+
const candidate = token.replace(SURROUNDING_PUNCTUATION, '').replace(SEARCH_OPERATOR_PREFIX, '');
111+
if (!candidate.includes('.')) return null;
112+
113+
try {
114+
return new URL(/^https?:\/\//i.test(candidate) ? candidate : `https://${candidate}`);
115+
} catch {
116+
return null;
117+
}
118+
}
119+
120+
function findRuleForUrl(url: URL): SiteRule | undefined {
121+
return SITE_RULES.find((rule) => {
122+
const matchesDomain = rule.domains.some((domain) => hostnameMatchesDomain(url.hostname, domain));
123+
return matchesDomain && (rule.matchesUrl?.(url) ?? true);
124+
});
125+
}
126+
127+
export function findActorTip(input: { query?: string; url?: string }): ActorTip | null {
128+
const target = input.query || input.url;
129+
if (!target) return null;
130+
131+
for (const token of target.split(/\s+/)) {
132+
if (token.startsWith('-')) continue;
133+
134+
const url = interpretTokenAsUrl(token);
135+
const rule = url ? findRuleForUrl(url) : undefined;
136+
if (!rule) continue;
137+
138+
return {
139+
message: `For scraping ${rule.site}, we recommend using [${rule.actorTitle}](${rule.actorUrl})`,
140+
level: 'info',
141+
};
142+
}
143+
144+
return null;
145+
}
146+
147+
/**
148+
* Stores the tip under the reserved key-value store key. Call it only once the crawlers have finished:
149+
* Crawlee purges the default storages while they start up, which wipes a record written before that.
150+
*
151+
* A tip is advisory, so a failure to store it never changes the outcome of the run.
152+
*/
153+
export async function storeActorTip(tip: ActorTip | null): Promise<void> {
154+
if (!tip) return;
155+
156+
try {
157+
await Actor.setValue(TIP_KVS_KEY, tip);
158+
} catch (err) {
159+
log.warning(`Failed to store the tip: ${err instanceof Error ? err.message : String(err)}`);
160+
}
161+
}

‎tests/tips.test.ts‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import { Actor } from 'apify';
2+
import { afterEach, describe, expect, it, vi } from 'vitest';
3+
4+
import { findActorTip, storeActorTip } from '../src/tips.js';
5+
6+
describe('findActorTip', () => {
7+
it.each([
8+
['https://www.facebook.com/apify/posts', 'For scraping facebook.com, we recommend using [Facebook Posts Scraper](https://apify.com/apify/facebook-posts-scraper)'],
9+
['https://www.facebook.com/groups/1234567890', 'For scraping Facebook groups, we recommend using [Facebook Groups Scraper](https://apify.com/apify/facebook-groups-scraper)'],
10+
['https://www.zillow.com/homedetails/123', 'For scraping zillow.com, we recommend using [Zillow Detail Scraper](https://apify.com/maxcopell/zillow-detail-scraper)'],
11+
['https://www.instagram.com/apify/', 'For scraping instagram.com, we recommend using [Instagram Scraper](https://apify.com/apify/instagram-scraper)'],
12+
['https://www.google.com/maps/place/Prague', 'For scraping Google Maps, we recommend using [Google Maps Scraper](https://apify.com/compass/crawler-google-places)'],
13+
['https://maps.google.com/?q=prague', 'For scraping Google Maps, we recommend using [Google Maps Scraper](https://apify.com/compass/crawler-google-places)'],
14+
['https://www.booking.com/hotel/cz/prague.html', 'For scraping booking.com, we recommend using [Booking Scraper](https://apify.com/voyager/booking-scraper)'],
15+
['https://www.tripadvisor.com/Hotel_Review-g274707', 'For scraping tripadvisor.com, we recommend using [Tripadvisor Scraper](https://apify.com/maxcopell/tripadvisor)'],
16+
['https://www.youtube.com/watch?v=dQw4w9WgXcQ', 'For scraping youtube.com, we recommend using [YouTube Scraper](https://apify.com/streamers/youtube-scraper)'],
17+
['https://youtu.be/dQw4w9WgXcQ', 'For scraping youtube.com, we recommend using [YouTube Scraper](https://apify.com/streamers/youtube-scraper)'],
18+
['https://www.tiktok.com/@apify', 'For scraping tiktok.com, we recommend using [TikTok Scraper](https://apify.com/clockworks/tiktok-scraper)'],
19+
['https://www.amazon.com/dp/B08N5WRWNW', 'For scraping amazon.com, we recommend using [Amazon Crawler](https://apify.com/junglee/amazon-crawler)'],
20+
])('recommends the Actor listed in the issue for %s', (query, message) => {
21+
expect(findActorTip({ query })?.message).toBe(message);
22+
});
23+
24+
it('marks the tip as informational', () => {
25+
expect(findActorTip({ query: 'https://www.tiktok.com/@apify' })?.level).toBe('info');
26+
});
27+
28+
it.each([
29+
// The site name appears in the path, not in the host.
30+
'https://example.com/facebook',
31+
'https://example.com/redirect?to=https://www.facebook.com/apify',
32+
// Plain Google Search is what this Actor does itself.
33+
'https://www.google.com/search?q=prague+restaurants',
34+
// Hosts that only look like a listed domain.
35+
'https://notfacebook.com/apify',
36+
'https://facebook.com.evil.example/apify',
37+
])('does not recommend anything for %s', (query) => {
38+
expect(findActorTip({ query })).toBeNull();
39+
});
40+
41+
it.each([
42+
['site:instagram.com nike', 'Instagram Scraper'],
43+
['cheap hotels booking.com prague', 'Booking Scraper'],
44+
['google.com/maps prague restaurants', 'Google Maps Scraper'],
45+
['"tripadvisor.com" prague', 'Tripadvisor Scraper'],
46+
])('finds a listed domain mentioned in the search query %s', (query, actorTitle) => {
47+
expect(findActorTip({ query })?.message).toContain(actorTitle);
48+
});
49+
50+
it.each([
51+
'best hotels in prague',
52+
'site:google.com apify',
53+
'nike -www.instagram.com',
54+
])('does not recommend anything for the search query %s', (query) => {
55+
expect(findActorTip({ query })).toBeNull();
56+
});
57+
58+
it.each([
59+
['https://www.amazon.co.uk/dp/B08N5WRWNW', 'Amazon Crawler'],
60+
['https://www.amazon.de/dp/B08N5WRWNW', 'Amazon Crawler'],
61+
['https://www.amazon.com.mx/dp/B08N5WRWNW', 'Amazon Crawler'],
62+
['https://www.google.co.uk/maps/place/Prague', 'Google Maps Scraper'],
63+
['https://maps.google.co.uk/?q=prague', 'Google Maps Scraper'],
64+
['https://www.tripadvisor.co.uk/Hotel_Review-g274707', 'Tripadvisor Scraper'],
65+
])('recognizes the country domain %s', (query, actorTitle) => {
66+
expect(findActorTip({ query })?.message).toContain(actorTitle);
67+
});
68+
69+
it.each([
70+
// The brand is only a subdomain of someone else's site.
71+
'https://amazon.abc.com/dp/1',
72+
'https://amazon.example.com/dp/1',
73+
'https://notamazon.de/dp/1',
74+
'https://google.com.evil.example/maps',
75+
// A country domain does not turn plain Google Search into a Maps request.
76+
'https://www.google.co.uk/search?q=prague',
77+
])('does not read %s as a country domain of a listed site', (query) => {
78+
expect(findActorTip({ query })).toBeNull();
79+
});
80+
81+
it('keeps sites that have a single global domain pinned to it', () => {
82+
expect(findActorTip({ query: 'https://facebook.ru/apify' })).toBeNull();
83+
expect(findActorTip({ query: 'https://instagram.de/apify' })).toBeNull();
84+
expect(findActorTip({ query: 'https://booking.de/hotel/cz/prague.html' })).toBeNull();
85+
});
86+
87+
it('matches subdomains, keeping the more specific rule first', () => {
88+
expect(findActorTip({ query: 'https://m.facebook.com/groups/123' })?.message).toContain('Facebook Groups Scraper');
89+
expect(findActorTip({ query: 'https://m.facebook.com/apify' })?.message).toContain('Facebook Posts Scraper');
90+
});
91+
92+
it('matches a path segment exactly, not merely its prefix', () => {
93+
expect(findActorTip({ query: 'https://www.facebook.com/groups' })?.message).toContain('Facebook Groups Scraper');
94+
expect(findActorTip({ query: 'https://www.google.com/maps' })?.message).toContain('Google Maps Scraper');
95+
expect(findActorTip({ query: 'https://www.facebook.com/groupsomething' })?.message).toContain('Facebook Posts Scraper');
96+
expect(findActorTip({ query: 'https://www.google.com/mapsomething' })).toBeNull();
97+
});
98+
99+
it.each([
100+
'go to zillow.com for listings',
101+
'go to zillow.com, and get the listings',
102+
'go to zillow.com. thanks',
103+
'go to zillow.com; now',
104+
'go to zillow.com!',
105+
'is zillow.com down?',
106+
'see (zillow.com) for details',
107+
'see [zillow.com] for details',
108+
'go to zillow.com...',
109+
])('finds the domain whatever punctuation surrounds it: %s', (query) => {
110+
expect(findActorTip({ query })?.message).toContain('Zillow Detail Scraper');
111+
});
112+
113+
it('strips the punctuation before matching a path segment', () => {
114+
expect(findActorTip({ query: 'google.com/maps, prague' })?.message).toContain('Google Maps Scraper');
115+
});
116+
117+
it('ignores the case of the input', () => {
118+
expect(findActorTip({ query: 'HTTPS://WWW.TIKTOK.COM/@APIFY' })?.message).toContain('TikTok Scraper');
119+
expect(findActorTip({ query: 'Site:Instagram.com nike' })?.message).toContain('Instagram Scraper');
120+
});
121+
122+
it('never throws on tokens that cannot be parsed as a URL', () => {
123+
expect(findActorTip({ query: 'https:// http://[ .. :: %% "" -' })).toBeNull();
124+
});
125+
126+
it('reads `query` first and falls back to `url`', () => {
127+
expect(findActorTip({ url: 'https://www.tiktok.com/@apify' })?.message).toContain('TikTok Scraper');
128+
expect(findActorTip({ query: '', url: 'https://www.tiktok.com/@apify' })?.message).toContain('TikTok Scraper');
129+
expect(findActorTip({ query: 'https://www.tiktok.com/@apify', url: 'https://www.amazon.com/dp/1' })?.message).toContain('TikTok Scraper');
130+
expect(findActorTip({})).toBeNull();
131+
});
132+
});
133+
134+
describe('storeActorTip', () => {
135+
afterEach(() => {
136+
vi.restoreAllMocks();
137+
});
138+
139+
it('stores the tip under the reserved `TIP` key', async () => {
140+
const setValue = vi.spyOn(Actor, 'setValue').mockResolvedValue(undefined);
141+
const tip = { message: 'For scraping tiktok.com, ...', level: 'info' } as const;
142+
143+
await storeActorTip(tip);
144+
145+
expect(setValue).toHaveBeenCalledWith('TIP', tip);
146+
});
147+
148+
it('writes nothing when there is no tip', async () => {
149+
const setValue = vi.spyOn(Actor, 'setValue').mockResolvedValue(undefined);
150+
151+
await storeActorTip(null);
152+
153+
expect(setValue).not.toHaveBeenCalled();
154+
});
155+
});

0 commit comments

Comments
 (0)