Fixes for false positive matchers and ontology

CONTRIBUTING.md

@@ -14,3 +14,61 @@ It will be better to make test in the same commit. You can do it the following w
 **End-to-end coverage:** each scheme (site/method) in `schemes.py` should have **at least one** e2e test in `tests/test_e2e.py` against a real URL or API response. Put the scheme name(s) in the test docstring (one per line) for `revision.py`. If the site is unreliable from CI, use `@pytest.mark.github_failed` or `rate_limited` (see [`docs/testing-and-ci.md`](docs/testing-and-ci.md)).
 
 And don't forget to update the table with methods by the script `./revision.py`!
+
+## Writing robust `flags`
+
+`flags` are substrings that **all** must appear in the response body for a scheme to
+match. They are the only gate — there is no URL check at extraction time.
+If flags are too generic, the scheme will fire on responses from unrelated sites
+and either produce garbage output or shadow the correct scheme (since `extract()`
+returns on the **first** match).
+
+### Rules
+
+1. **At least one flag must be unique to the platform.** Good examples:
+   `'OK.startupData'`, `'canonicalPeriscopeUrl'`, `'data-initial-data='`.
+   Bad: `'"data"'`, `'"user"'`, `'"username"'` — these match any JSON API.
+
+2. **Prefer structural API field names** that only this site returns:
+   `'"allowCrawler"'` (Wattpad), `'"dateJoined"'` + `'"socialMediaLinks"'` (hashnode),
+   `'"creatorTraders"'` (Manifold). These survive redesigns.
+
+3. **Never use a single short JSON key as the only flag.**
+   `'{"username":"'` alone matches dozens of APIs. Add a second flag that is
+   specific to the platform.
+
+4. **For HTML pages, use CSS class names or page-specific markers** instead of
+   generic tags: `'osu-layout'`, `'ProfileHeader_lblMemberName'`,
+   `'Aedu.User.set_viewed('`.
+
+5. **For RSC / escaped JSON**, remember that flags check the raw response body.
+   Strings appear as `\"field_name\"`, not `"field_name"`. Prefer unescaped
+   markers from the surrounding HTML (`'op.gg/lol/summoners/'`).
+
+6. **Test your flags against 5–10 other sites' responses** (run
+   `maigret USER --site "YourSite" -vvv` and check `debug.log` for false
+   triggers). A scheme that fires once for its target and zero times for
+   others is correct.
+
+### Quick checklist
+
+| Good flag | Why |
+|-----------|-----|
+| `'data-initial-data='` | HTML attribute unique to osu! |
+| `'"profilesData.profileUser"'` | JS variable unique to GOG |
+| `'"allowCrawler"'` | JSON field unique to Wattpad API |
+| `'"dateJoined"', '"socialMediaLinks"'` | Two fields unique to hashnode |
+| `'Music Profile \| Last.fm</title>'` | Title tag with site name |
+
+| Bad flag | Problem |
+|----------|---------|
+| `'"data"'` | Matches any JSON |
+| `'"user"'` | Matches any user API |
+| `'{"username":"'` | Matches any JSON with username |
+| `'__NEXT_DATA__'` (alone) | Matches any Next.js site |
+
+### Field naming
+
+Use standard names from [`FIELDS.md`](FIELDS.md). Platform-specific fields get
+a platform prefix (`osu_pp`, `gog_games_owned`). See FIELDS.md for the complete
+ontology.

FIELDS.md

@@ -25,13 +25,17 @@ If `name` is a login/handle, map it to `username`.
 | `image_bg` | Background / banner image URL | `banner`, `cover`, `bannerImageURL`, `cover_250_url` |
 | `website` | Personal website URL | `url`, `domain_url`, `external_url`, `blog_url` |
 | `email` | Public email | -- |
+| `occupation` | Job title / profession | `jobTitle`, `role`, `work` |
+| `company` | Employer / organization | `company_name`, `organization`, `worksFor` |
+| `interests` | Interests / hobbies (free text) | `interest_names` |
 
 ## Demographics
 
 | Field | Description | API mapping examples |
 |-------|-------------|----------------------|
 | `gender` | Gender | `sex` |
-| `country` | Country | `country_code` (normalize to name) |
+| `country` | Country (name) | `country_code` (normalize to name) |
+| `country_code` | Country (ISO 3166-1 alpha-2 code) | `countryCode`, `country_code` |
 | `city` | City | -- |
 | `location` | Location (free text, city+country) | `address` |
 | `birthday` | Date of birth | `birth_date`, `dateOfBirth` |

socid_extractor/schemes.py

@@ -284,7 +284,7 @@ def _virgool_links(user_row):
             'fullname': lambda x: x['photostream-models'][0]['owner'].get('realname'),
             'location': lambda x: x['person-profile-models'][0].get('location'),
             'image': lambda x: 'https:' + x['photostream-models'][0]['owner']['buddyicon']['retina'],
-            'photo_count': lambda x: x['person-profile-models'][0]['photoCount'],
+            'photos_count': lambda x: x['person-profile-models'][0]['photoCount'],
             'follower_count': lambda x: x['person-contacts-count-models'][0]['followerCount'],
             'following_count': lambda x: x['person-contacts-count-models'][0]['followingCount'],
             'created_at': lambda x: parse_datetime(x['photostream-models'][0]['owner'].get('dateCreated', 0)),
@@ -1627,9 +1627,9 @@ def _virgool_links(user_row):
         'bs': True,
         'fields': {
             'fullname': lambda x: x.find('span', {'class': 'header-title-display-name'}).contents[0].strip(),
-            # TODO: date convert
             'bio': lambda x: x.find('span', {'class': 'header-scrobble-since'}).contents[0].strip(),
             'image': lambda x: x.find('span', {'class': 'avatar'}).find('img').get('src', ''),
+            'created_at': lambda x: (lambda m: m.group(1) if m else None)(re.search(r'(\d{4})', x.find('span', {'class': 'header-scrobble-since'}).text)) if x.find('span', {'class': 'header-scrobble-since'}) else None,
         }
     },
     'Ask.fm': {
@@ -1671,7 +1671,7 @@ def _virgool_links(user_row):
             'image': lambda x: x.find('div', {'class': 'authorBlock-avatar'}).find('img').get('src', ''),
             'bio': lambda x: '\n'.join(x.find('p', {'class': 'authorBlock-header-bio'}).contents),
             'links': lambda x: [a.get('href') for a in x.find('div', {'class': 'authorBlock-meta'}).findAll('a')],
-            'joined_year': lambda x: extract_digits(
+            'created_at': lambda x: extract_digits(
                 x.find('div', {'class': 'authorBlock-header'}).find('h6').contents[0]),
         }
     },
@@ -1745,15 +1745,15 @@ def _virgool_links(user_row):
             'image': lambda x: get_ucoz_image(x),
             'gender': lambda x: x.find('div', string='Имя:').next_sibling.split(' ')[-2],
             'created_at': lambda x: x.find('div', string='Дата регистрации:').next_sibling.strip(),
-            'last_seen_at': lambda x: x.find('div', string='Дата входа:').next_sibling.strip(),
+            'latest_activity_at': lambda x: x.find('div', string='Дата входа:').next_sibling.strip(),
             'link': lambda x: get_ucoz_uid_node(x).parent.get('href'),
             'uidme_uguid': lambda x: get_ucoz_uid_node(x).parent.get('href', '').split('/')[-1],
             'location': lambda x: x.find('div', string='Место проживания:').next_sibling.strip(),
             'country': lambda x: x.find('div', string='Страна:').next_sibling.strip(),
             'city': lambda x: x.find('div', string='Город:').next_sibling.strip(),
             'state': lambda x: x.find('div', string='Штат:').next_sibling.strip(),
             'email': lambda x: get_ucoz_email(x.find('div', string='E-mail:').next_sibling.strip()),
-            'birthday_at': lambda x: x.find('div', string='Дата рождения:').next_sibling.split('[')[0].strip(),
+            'birthday': lambda x: x.find('div', string='Дата рождения:').next_sibling.split('[')[0].strip(),
         },
     },
     'uID.me': {
@@ -1954,9 +1954,9 @@ def _virgool_links(user_row):
             'broadcasts_count': lambda x: x.get('n_broadcasts'),
             'is_beta_user': lambda x: x['is_beta_user'],
             'is_employee': lambda x: x['is_employee'],
-            'isVerified': lambda x: x['isVerified'],
+            'is_verified': lambda x: x['isVerified'],
             'is_twitter_verified': lambda x: x['is_twitter_verified'],
-            'twitterUserId': lambda x: x.get('twitterUserId'),
+            'twitter_uid': lambda x: x.get('twitterUserId'),
             'twitter_screen_name': lambda x: x.get('twitter_screen_name'),
             'image': lambda x: x['profile_image_urls'][0]['url'],
         }
@@ -2046,7 +2046,7 @@ def _virgool_links(user_row):
             'image': lambda x: x['avatar']['url'],
             'follower_count': lambda x: x['num']['subscriptions'],
             'following_count': lambda x: x['num']['subscribers'],
-            'post_count': lambda x: x['num']['total_posts'],
+            'posts_count': lambda x: x['num']['total_posts'],
             'created_count': lambda x: x['num']['created'],
             'featured_count': lambda x: x['num']['featured'],
             'smile_count': lambda x: x['num']['total_smiles'],
@@ -2056,7 +2056,7 @@ def _virgool_links(user_row):
     },
     'Wattpad API': {
         'url_hints': ('wattpad.com',),
-        'flags': ['{"username":"'],
+        'flags': ['{"username":"', '"allowCrawler"'],
         'regex': r'^({"username":"(.+)})$',
         'extract_json': True,
         'url_mutations': [
@@ -2187,8 +2187,8 @@ def _virgool_links(user_row):
             'twitter_url': lambda x: x['user'].get('twitterHandle'),
             'linkedin_url': lambda x: x['user'].get('linkedinHandle'),
             'links': lambda x: x['user'].get('personalWebsite'),
-            'isAdmin': lambda x: x['user'].get('isAdmin'),
-            'isVerified': lambda x: x['user'].get('isVerified'),
+            'is_admin': lambda x: x['user'].get('isAdmin'),
+            'is_verified': lambda x: x['user'].get('isVerified'),
             'HistoryPublic': lambda x: x['user'].get('preferredHistoryPublic'),
             'RoomPublic': lambda x: x['user'].get('preferredRoomPublic'),
             'InviteOnly': lambda x: x['user'].get('preferredInviteOnly'),
@@ -2470,16 +2470,21 @@ def _virgool_links(user_row):
     },
     'hashnode GraphQL API': {
         'url_hints': ('hashnode.com', 'gql.hashnode.com'),
-        'flags': ['"data"', '"user"'],
+        'flags': ['"dateJoined"', '"socialMediaLinks"'],
         'regex': r'^(\{[\s\S]*\})$',
         'extract_json': True,
         'fields': {
             'username': lambda x: x.get('data', {}).get('user', {}).get('username') if x.get('data', {}).get('user') else None,
             'fullname': lambda x: x.get('data', {}).get('user', {}).get('name') if x.get('data', {}).get('user') else None,
+            'bio': lambda x: x.get('data', {}).get('user', {}).get('tagline') or None if x.get('data', {}).get('user') else None,
+            'created_at': lambda x: x.get('data', {}).get('user', {}).get('dateJoined') if x.get('data', {}).get('user') else None,
+            'twitter_username': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('twitter', '').rstrip('/').rsplit('/', 1)[-1] or None if x.get('data', {}).get('user') else None,
+            'github_username': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('github', '').rstrip('/').rsplit('/', 1)[-1] or None if x.get('data', {}).get('user') else None,
+            'website': lambda x: (x.get('data', {}).get('user', {}).get('socialMediaLinks', {}) or {}).get('website') or None if x.get('data', {}).get('user') else None,
         },
         'url_mutations': [{
             'from': r'https?://hashnode\.com/@(?P<username>[^/?#]+)',
-            'to': 'https://gql.hashnode.com?query=%7Buser(username%3A%20%22{username}%22)%20%7B%20name%20username%20%7D%7D',
+            'to': 'https://gql.hashnode.com?query=%7Buser(username%3A%20%22{username}%22)%20%7B%20name%20username%20tagline%20dateJoined%20socialMediaLinks%20%7B%20twitter%20github%20linkedin%20website%20%7D%20%7D%7D',
         }],
     },
     'Rarible API': {

tests/test_e2e.py

@@ -896,7 +896,7 @@ def test_flickr():
     assert info.get('fullname') == 'alexaim%E9 photography'
     assert info.get(
         'image') == 'https://farm66.staticflickr.com/65535/buddyicons/187482857@N04_r.jpg?1584445364#187482857@N04'
-    assert int(info.get('photo_count')) > 140
+    assert int(info.get('photos_count')) > 140
     assert int(info.get('follower_count')) > 180
     assert int(info.get('following_count')) > 70
     assert info.get('created_at').startswith('2020-03-17')
@@ -943,6 +943,7 @@ def test_last_fm():
 
     assert info.get('fullname') == 'Alex'
     assert info.get('bio') == '• scrobbling since 21 Feb 2003'
+    assert info.get('created_at') == '2003'
     assert info.get('image') == 'https://lastfm.freetls.fastly.net/i/u/avatar170s/15e455555655c8503ed9ba6fce71d2d6.png'
 
 
@@ -1011,7 +1012,7 @@ def test_xakep():
     assert info.get(
         'bio') == 'Координатор проекта VyOS (https://vyos.io), «языковед», функциональщик,  иногда сетевой администратор'
     assert info.get('links') == "['https://www.baturin.org']"
-    assert info.get('joined_year') == '2018'
+    assert info.get('created_at') == '2018'
     assert info.get('gravatar_url') == 'https://gravatar.com/b1859c813547de1bba3c65bc4b1a217c'
     assert info.get('gravatar_username') == 'https://gravatar.com/b1859c813547de1bba3c65bc4b1a217c'
     assert info.get('gravatar_email_md5_hash') == 'b1859c813547de1bba3c65bc4b1a217c'
@@ -1066,12 +1067,12 @@ def test_ucoz_1():
     assert info.get('fullname') == 'Михаил ко'
     assert info.get('gender') == 'Мужчина'
     assert info.get('created_at') == 'Пятница, 23.01.2015, 15:02'
-    assert info.get('last_seen_at') == 'Пятница, 23.01.2015, 15:07'
+    assert info.get('latest_activity_at') == 'Пятница, 23.01.2015, 15:07'
     # uid.me deep link no longer present in static HTML (2026)
     assert info.get('location') == 'Российская Федерация'
     assert info.get('city') == 'Москва'
     assert info.get('state') == 'Москва'
-    assert info.get('birthday_at') == '16 Декабря 1971'
+    assert info.get('birthday') == '16 Декабря 1971'
 
 
 @pytest.mark.skip(reason="thaicat.ru often unreachable / connect timeout from CI and local (2026)")
@@ -1083,10 +1084,10 @@ def test_ucoz_2():
     assert info.get('image') == 'http://www.thaicat.ru/avatar/00/20/419858.jpg'
     assert info.get('gender') == 'Женщина'
     assert info.get('created_at') == 'Суббота, 14.01.2012, 17:41'
-    assert info.get('last_seen_at') == 'Суббота, 14.01.2012, 17:41'
+    assert info.get('latest_activity_at') == 'Суббота, 14.01.2012, 17:41'
     assert info.get('country') == 'Италия'
     assert info.get('city') == 'l\'aquila'
-    assert info.get('birthday_at') == '10 Июля 1975'
+    assert info.get('birthday') == '10 Июля 1975'
 
 
 def test_ucoz_3():
@@ -1095,7 +1096,7 @@ def test_ucoz_3():
     assert info.get('url') == 'https://prenatal-club.ucoz.com/index/8-128'
     assert info.get('image') == 'https://425523249.uid.me/avatar.jpg'
     assert info.get('created_at') == 'Среда, 10.03.2010, 09:42'
-    assert info.get('last_seen_at') == 'Среда, 10.03.2010, 09:42'
+    assert info.get('latest_activity_at') == 'Среда, 10.03.2010, 09:42'
     # uid.me deep link no longer present in static HTML (2026)
     assert info.get('location') == 'Российская Федерация'
     assert info.get('city') == 'Санкт-Петербург'
@@ -1288,7 +1289,7 @@ def test_ifunny_co():
     assert info.get("image", "").startswith("https://imageproxy.ifunny.co/noop/user_photos/")
     # assert int(info.get("follower_count")) >= 0
     # assert int(info.get("following_count")) >= 70
-    # assert int(info.get("post_count")) >= 127
+    # assert int(info.get("posts_count")) >= 127
     # assert int(info.get("created_count")) >= 127
     # assert info.get("featured_count") == "7"
     # assert int(info.get("smile_count")) > 32000
@@ -1362,8 +1363,8 @@ def test_binarysearch_api():  # Broken. Site is not responding.
     assert info.get("location") == "New York, NY, USA"
     assert info.get("bio") == "This is fun."
     assert info.get("links") == "https://www.youtube.com/c/Algorithmist/"
-    assert info.get("isAdmin") == "False"
-    assert info.get("isVerified") == "True"
+    assert info.get("is_admin") == "False"
+    assert info.get("is_verified") == "True"
     assert info.get("HistoryPublic") == "False"
     assert info.get("RoomPublic") == "True"
     assert info.get("InviteOnly") == "False"

tests/test_socid_improvements.py

@@ -400,20 +400,33 @@ def test_hashnode_graphql_api_json():
         "data": {
             "user": {
                 "name": "Melwin D'Almeida",
-                "username": "melwinalm"
+                "username": "melwinalm",
+                "tagline": "Cloud enthusiast",
+                "dateJoined": "2018-02-18T15:24:29.694Z",
+                "socialMediaLinks": {
+                    "twitter": "https://twitter.com/melwinalm",
+                    "github": "",
+                    "linkedin": None,
+                    "website": ""
+                }
             }
         }
     })
     info = extract(body)
     assert info.get('username') == 'melwinalm'
     assert info.get('fullname') == "Melwin D'Almeida"
+    assert info.get('bio') == 'Cloud enthusiast'
+    assert info.get('created_at') == '2018-02-18T15:24:29.694Z'
+    assert info.get('twitter_username') == 'melwinalm'
 
 
 def test_hashnode_graphql_api_null_user():
     """hashnode GraphQL API: null user (unclaimed) should yield empty result."""
     body = json.dumps({
         "data": {
-            "user": None
+            "user": None,
+            "dateJoined": None,
+            "socialMediaLinks": None
         }
     })
     info = extract(body)
@@ -538,9 +551,9 @@ def test_periscope_profile_extraction():
     assert info.get('broadcasts_count') == '42'
     assert info.get('is_beta_user') == 'False'
     assert info.get('is_employee') == 'False'
-    assert info.get('isVerified') == 'False'
+    assert info.get('is_verified') == 'False'
     assert info.get('is_twitter_verified') == 'True'
-    assert info.get('twitterUserId') == '78901234'
+    assert info.get('twitter_uid') == '78901234'
     assert info.get('twitter_screen_name') == 'polina_z'
     assert info.get('created_at') == '2016-04-10T18:22:05.411012300+00:00'