diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3272120..246013b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,3 +14,61 @@ It will be better to make test in the same commit. You can do it the following w **End-to-end coverage:** each scheme (site/method) in `schemes.py` should have **at least one** e2e test in `tests/test_e2e.py` against a real URL or API response. Put the scheme name(s) in the test docstring (one per line) for `revision.py`. If the site is unreliable from CI, use `@pytest.mark.github_failed` or `rate_limited` (see [`docs/testing-and-ci.md`](docs/testing-and-ci.md)). And don't forget to update the table with methods by the script `./revision.py`! + +## Writing robust `flags` + +`flags` are substrings that **all** must appear in the response body for a scheme to +match. They are the only gate — there is no URL check at extraction time. +If flags are too generic, the scheme will fire on responses from unrelated sites +and either produce garbage output or shadow the correct scheme (since `extract()` +returns on the **first** match). + +### Rules + +1. **At least one flag must be unique to the platform.** Good examples: + `'OK.startupData'`, `'canonicalPeriscopeUrl'`, `'data-initial-data='`. + Bad: `'"data"'`, `'"user"'`, `'"username"'` — these match any JSON API. + +2. **Prefer structural API field names** that only this site returns: + `'"allowCrawler"'` (Wattpad), `'"dateJoined"'` + `'"socialMediaLinks"'` (hashnode), + `'"creatorTraders"'` (Manifold). These survive redesigns. + +3. **Never use a single short JSON key as the only flag.** + `'{"username":"'` alone matches dozens of APIs. Add a second flag that is + specific to the platform. + +4. **For HTML pages, use CSS class names or page-specific markers** instead of + generic tags: `'osu-layout'`, `'ProfileHeader_lblMemberName'`, + `'Aedu.User.set_viewed('`. + +5. **For RSC / escaped JSON**, remember that flags check the raw response body. + Strings appear as `\"field_name\"`, not `"field_name"`. Prefer unescaped + markers from the surrounding HTML (`'op.gg/lol/summoners/'`). + +6. **Test your flags against 5–10 other sites' responses** (run + `maigret USER --site "YourSite" -vvv` and check `debug.log` for false + triggers). A scheme that fires once for its target and zero times for + others is correct. + +### Quick checklist + +| Good flag | Why | +|-----------|-----| +| `'data-initial-data='` | HTML attribute unique to osu! | +| `'"profilesData.profileUser"'` | JS variable unique to GOG | +| `'"allowCrawler"'` | JSON field unique to Wattpad API | +| `'"dateJoined"', '"socialMediaLinks"'` | Two fields unique to hashnode | +| `'Music Profile \| Last.fm'` | Title tag with site name | + +| Bad flag | Problem | +|----------|---------| +| `'"data"'` | Matches any JSON | +| `'"user"'` | Matches any user API | +| `'{"username":"'` | Matches any JSON with username | +| `'__NEXT_DATA__'` (alone) | Matches any Next.js site | + +### Field naming + +Use standard names from [`FIELDS.md`](FIELDS.md). Platform-specific fields get +a platform prefix (`osu_pp`, `gog_games_owned`). See FIELDS.md for the complete +ontology. diff --git a/FIELDS.md b/FIELDS.md index a7efa1d..613b315 100644 --- a/FIELDS.md +++ b/FIELDS.md @@ -25,13 +25,17 @@ If `name` is a login/handle, map it to `username`. | `image_bg` | Background / banner image URL | `banner`, `cover`, `bannerImageURL`, `cover_250_url` | | `website` | Personal website URL | `url`, `domain_url`, `external_url`, `blog_url` | | `email` | Public email | -- | +| `occupation` | Job title / profession | `jobTitle`, `role`, `work` | +| `company` | Employer / organization | `company_name`, `organization`, `worksFor` | +| `interests` | Interests / hobbies (free text) | `interest_names` | ## Demographics | Field | Description | API mapping examples | |-------|-------------|----------------------| | `gender` | Gender | `sex` | -| `country` | Country | `country_code` (normalize to name) | +| `country` | Country (name) | `country_code` (normalize to name) | +| `country_code` | Country (ISO 3166-1 alpha-2 code) | `countryCode`, `country_code` | | `city` | City | -- | | `location` | Location (free text, city+country) | `address` | | `birthday` | Date of birth | `birth_date`, `dateOfBirth` | diff --git a/socid_extractor/schemes.py b/socid_extractor/schemes.py index 5b73c8e..3c050cd 100644 --- a/socid_extractor/schemes.py +++ b/socid_extractor/schemes.py @@ -284,7 +284,7 @@ def _virgool_links(user_row): 'fullname': lambda x: x['photostream-models'][0]['owner'].get('realname'), 'location': lambda x: x['person-profile-models'][0].get('location'), 'image': lambda x: 'https:' + x['photostream-models'][0]['owner']['buddyicon']['retina'], - 'photo_count': lambda x: x['person-profile-models'][0]['photoCount'], + 'photos_count': lambda x: x['person-profile-models'][0]['photoCount'], 'follower_count': lambda x: x['person-contacts-count-models'][0]['followerCount'], 'following_count': lambda x: x['person-contacts-count-models'][0]['followingCount'], 'created_at': lambda x: parse_datetime(x['photostream-models'][0]['owner'].get('dateCreated', 0)), @@ -1627,9 +1627,9 @@ def _virgool_links(user_row): 'bs': True, 'fields': { 'fullname': lambda x: x.find('span', {'class': 'header-title-display-name'}).contents[0].strip(), - # TODO: date convert 'bio': lambda x: x.find('span', {'class': 'header-scrobble-since'}).contents[0].strip(), 'image': lambda x: x.find('span', {'class': 'avatar'}).find('img').get('src', ''), + 'created_at': lambda x: (lambda m: m.group(1) if m else None)(re.search(r'(\d{4})', x.find('span', {'class': 'header-scrobble-since'}).text)) if x.find('span', {'class': 'header-scrobble-since'}) else None, } }, 'Ask.fm': { @@ -1671,7 +1671,7 @@ def _virgool_links(user_row): 'image': lambda x: x.find('div', {'class': 'authorBlock-avatar'}).find('img').get('src', ''), 'bio': lambda x: '\n'.join(x.find('p', {'class': 'authorBlock-header-bio'}).contents), 'links': lambda x: [a.get('href') for a in x.find('div', {'class': 'authorBlock-meta'}).findAll('a')], - 'joined_year': lambda x: extract_digits( + 'created_at': lambda x: extract_digits( x.find('div', {'class': 'authorBlock-header'}).find('h6').contents[0]), } }, @@ -1745,7 +1745,7 @@ def _virgool_links(user_row): 'image': lambda x: get_ucoz_image(x), 'gender': lambda x: x.find('div', string='Имя:').next_sibling.split(' ')[-2], 'created_at': lambda x: x.find('div', string='Дата регистрации:').next_sibling.strip(), - 'last_seen_at': lambda x: x.find('div', string='Дата входа:').next_sibling.strip(), + 'latest_activity_at': lambda x: x.find('div', string='Дата входа:').next_sibling.strip(), 'link': lambda x: get_ucoz_uid_node(x).parent.get('href'), 'uidme_uguid': lambda x: get_ucoz_uid_node(x).parent.get('href', '').split('/')[-1], 'location': lambda x: x.find('div', string='Место проживания:').next_sibling.strip(), @@ -1753,7 +1753,7 @@ def _virgool_links(user_row): 'city': lambda x: x.find('div', string='Город:').next_sibling.strip(), 'state': lambda x: x.find('div', string='Штат:').next_sibling.strip(), 'email': lambda x: get_ucoz_email(x.find('div', string='E-mail:').next_sibling.strip()), - 'birthday_at': lambda x: x.find('div', string='Дата рождения:').next_sibling.split('[')[0].strip(), + 'birthday': lambda x: x.find('div', string='Дата рождения:').next_sibling.split('[')[0].strip(), }, }, 'uID.me': { @@ -1954,9 +1954,9 @@ def _virgool_links(user_row): 'broadcasts_count': lambda x: x.get('n_broadcasts'), 'is_beta_user': lambda x: x['is_beta_user'], 'is_employee': lambda x: x['is_employee'], - 'isVerified': lambda x: x['isVerified'], + 'is_verified': lambda x: x['isVerified'], 'is_twitter_verified': lambda x: x['is_twitter_verified'], - 'twitterUserId': lambda x: x.get('twitterUserId'), + 'twitter_uid': lambda x: x.get('twitterUserId'), 'twitter_screen_name': lambda x: x.get('twitter_screen_name'), 'image': lambda x: x['profile_image_urls'][0]['url'], } @@ -2046,7 +2046,7 @@ def _virgool_links(user_row): 'image': lambda x: x['avatar']['url'], 'follower_count': lambda x: x['num']['subscriptions'], 'following_count': lambda x: x['num']['subscribers'], - 'post_count': lambda x: x['num']['total_posts'], + 'posts_count': lambda x: x['num']['total_posts'], 'created_count': lambda x: x['num']['created'], 'featured_count': lambda x: x['num']['featured'], 'smile_count': lambda x: x['num']['total_smiles'], @@ -2056,7 +2056,7 @@ def _virgool_links(user_row): }, 'Wattpad API': { 'url_hints': ('wattpad.com',), - 'flags': ['{"username":"'], + 'flags': ['{"username":"', '"allowCrawler"'], 'regex': r'^({"username":"(.+)})$', 'extract_json': True, 'url_mutations': [ @@ -2187,8 +2187,8 @@ def _virgool_links(user_row): 'twitter_url': lambda x: x['user'].get('twitterHandle'), 'linkedin_url': lambda x: x['user'].get('linkedinHandle'), 'links': lambda x: x['user'].get('personalWebsite'), - 'isAdmin': lambda x: x['user'].get('isAdmin'), - 'isVerified': lambda x: x['user'].get('isVerified'), + 'is_admin': lambda x: x['user'].get('isAdmin'), + 'is_verified': lambda x: x['user'].get('isVerified'), 'HistoryPublic': lambda x: x['user'].get('preferredHistoryPublic'), 'RoomPublic': lambda x: x['user'].get('preferredRoomPublic'), 'InviteOnly': lambda x: x['user'].get('preferredInviteOnly'), @@ -2470,16 +2470,21 @@ def _virgool_links(user_row): }, 'hashnode GraphQL API': { 'url_hints': ('hashnode.com', 'gql.hashnode.com'), - 'flags': ['"data"', '"user"'], + 'flags': ['"dateJoined"', '"socialMediaLinks"'], 'regex': r'^(\{[\s\S]*\})$', 'extract_json': True, 'fields': { 'username': lambda x: x.get('data', {}).get('user', {}).get('username') if x.get('data', {}).get('user') else None, 'fullname': lambda x: x.get('data', {}).get('user', {}).get('name') if x.get('data', {}).get('user') else None, + 'bio': lambda x: x.get('data', {}).get('user', {}).get('tagline') or None if x.get('data', {}).get('user') else None, + 'created_at': lambda x: x.get('data', {}).get('user', {}).get('dateJoined') if x.get('data', {}).get('user') else None, + 'twitter_username': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('twitter', '').rstrip('/').rsplit('/', 1)[-1] or None if x.get('data', {}).get('user') else None, + 'github_username': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('github', '').rstrip('/').rsplit('/', 1)[-1] or None if x.get('data', {}).get('user') else None, + 'website': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('website') or None if x.get('data', {}).get('user') else None, }, 'url_mutations': [{ 'from': r'https?://hashnode\.com/@(?P[^/?#]+)', - 'to': 'https://gql.hashnode.com?query=%7Buser(username%3A%20%22{username}%22)%20%7B%20name%20username%20%7D%7D', + 'to': 'https://gql.hashnode.com?query=%7Buser(username%3A%20%22{username}%22)%20%7B%20name%20username%20tagline%20dateJoined%20socialMediaLinks%20%7B%20twitter%20github%20linkedin%20website%20%7D%20%7D%7D', }], }, 'Rarible API': { diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 5ab629c..e06a1cd 100755 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -896,7 +896,7 @@ def test_flickr(): assert info.get('fullname') == 'alexaim%E9 photography' assert info.get( 'image') == 'https://farm66.staticflickr.com/65535/buddyicons/187482857@N04_r.jpg?1584445364#187482857@N04' - assert int(info.get('photo_count')) > 140 + assert int(info.get('photos_count')) > 140 assert int(info.get('follower_count')) > 180 assert int(info.get('following_count')) > 70 assert info.get('created_at').startswith('2020-03-17') @@ -943,6 +943,7 @@ def test_last_fm(): assert info.get('fullname') == 'Alex' assert info.get('bio') == '• scrobbling since 21 Feb 2003' + assert info.get('created_at') == '2003' assert info.get('image') == 'https://lastfm.freetls.fastly.net/i/u/avatar170s/15e455555655c8503ed9ba6fce71d2d6.png' @@ -1011,7 +1012,7 @@ def test_xakep(): assert info.get( 'bio') == 'Координатор проекта VyOS (https://vyos.io), «языковед», функциональщик, иногда сетевой администратор' assert info.get('links') == "['https://www.baturin.org']" - assert info.get('joined_year') == '2018' + assert info.get('created_at') == '2018' assert info.get('gravatar_url') == 'https://gravatar.com/b1859c813547de1bba3c65bc4b1a217c' assert info.get('gravatar_username') == 'https://gravatar.com/b1859c813547de1bba3c65bc4b1a217c' assert info.get('gravatar_email_md5_hash') == 'b1859c813547de1bba3c65bc4b1a217c' @@ -1066,12 +1067,12 @@ def test_ucoz_1(): assert info.get('fullname') == 'Михаил ко' assert info.get('gender') == 'Мужчина' assert info.get('created_at') == 'Пятница, 23.01.2015, 15:02' - assert info.get('last_seen_at') == 'Пятница, 23.01.2015, 15:07' + assert info.get('latest_activity_at') == 'Пятница, 23.01.2015, 15:07' # uid.me deep link no longer present in static HTML (2026) assert info.get('location') == 'Российская Федерация' assert info.get('city') == 'Москва' assert info.get('state') == 'Москва' - assert info.get('birthday_at') == '16 Декабря 1971' + assert info.get('birthday') == '16 Декабря 1971' @pytest.mark.skip(reason="thaicat.ru often unreachable / connect timeout from CI and local (2026)") @@ -1083,10 +1084,10 @@ def test_ucoz_2(): assert info.get('image') == 'http://www.thaicat.ru/avatar/00/20/419858.jpg' assert info.get('gender') == 'Женщина' assert info.get('created_at') == 'Суббота, 14.01.2012, 17:41' - assert info.get('last_seen_at') == 'Суббота, 14.01.2012, 17:41' + assert info.get('latest_activity_at') == 'Суббота, 14.01.2012, 17:41' assert info.get('country') == 'Италия' assert info.get('city') == 'l\'aquila' - assert info.get('birthday_at') == '10 Июля 1975' + assert info.get('birthday') == '10 Июля 1975' def test_ucoz_3(): @@ -1095,7 +1096,7 @@ def test_ucoz_3(): assert info.get('url') == 'https://prenatal-club.ucoz.com/index/8-128' assert info.get('image') == 'https://425523249.uid.me/avatar.jpg' assert info.get('created_at') == 'Среда, 10.03.2010, 09:42' - assert info.get('last_seen_at') == 'Среда, 10.03.2010, 09:42' + assert info.get('latest_activity_at') == 'Среда, 10.03.2010, 09:42' # uid.me deep link no longer present in static HTML (2026) assert info.get('location') == 'Российская Федерация' assert info.get('city') == 'Санкт-Петербург' @@ -1288,7 +1289,7 @@ def test_ifunny_co(): assert info.get("image", "").startswith("https://imageproxy.ifunny.co/noop/user_photos/") # assert int(info.get("follower_count")) >= 0 # assert int(info.get("following_count")) >= 70 - # assert int(info.get("post_count")) >= 127 + # assert int(info.get("posts_count")) >= 127 # assert int(info.get("created_count")) >= 127 # assert info.get("featured_count") == "7" # assert int(info.get("smile_count")) > 32000 @@ -1362,8 +1363,8 @@ def test_binarysearch_api(): # Broken. Site is not responding. assert info.get("location") == "New York, NY, USA" assert info.get("bio") == "This is fun." assert info.get("links") == "https://www.youtube.com/c/Algorithmist/" - assert info.get("isAdmin") == "False" - assert info.get("isVerified") == "True" + assert info.get("is_admin") == "False" + assert info.get("is_verified") == "True" assert info.get("HistoryPublic") == "False" assert info.get("RoomPublic") == "True" assert info.get("InviteOnly") == "False" diff --git a/tests/test_socid_improvements.py b/tests/test_socid_improvements.py index 4b51183..adaaaba 100644 --- a/tests/test_socid_improvements.py +++ b/tests/test_socid_improvements.py @@ -400,20 +400,33 @@ def test_hashnode_graphql_api_json(): "data": { "user": { "name": "Melwin D'Almeida", - "username": "melwinalm" + "username": "melwinalm", + "tagline": "Cloud enthusiast", + "dateJoined": "2018-02-18T15:24:29.694Z", + "socialMediaLinks": { + "twitter": "https://twitter.com/melwinalm", + "github": "", + "linkedin": None, + "website": "" + } } } }) info = extract(body) assert info.get('username') == 'melwinalm' assert info.get('fullname') == "Melwin D'Almeida" + assert info.get('bio') == 'Cloud enthusiast' + assert info.get('created_at') == '2018-02-18T15:24:29.694Z' + assert info.get('twitter_username') == 'melwinalm' def test_hashnode_graphql_api_null_user(): """hashnode GraphQL API: null user (unclaimed) should yield empty result.""" body = json.dumps({ "data": { - "user": None + "user": None, + "dateJoined": None, + "socialMediaLinks": None } }) info = extract(body) @@ -538,9 +551,9 @@ def test_periscope_profile_extraction(): assert info.get('broadcasts_count') == '42' assert info.get('is_beta_user') == 'False' assert info.get('is_employee') == 'False' - assert info.get('isVerified') == 'False' + assert info.get('is_verified') == 'False' assert info.get('is_twitter_verified') == 'True' - assert info.get('twitterUserId') == '78901234' + assert info.get('twitter_uid') == '78901234' assert info.get('twitter_screen_name') == 'polina_z' assert info.get('created_at') == '2016-04-10T18:22:05.411012300+00:00'