From 4717a2530c5d281133f0e971897f7d6d1ffe18c4 Mon Sep 17 00:00:00 2001 From: TecnoCreeper Date: Wed, 29 Jul 2026 20:14:56 +0200 Subject: [PATCH] Threads: edited scheme to extract more data and updated inline fixture test --- socid_extractor/schemes.py | 13 ++++- tests/test_socid_improvements.py | 95 ++++---------------------------- 2 files changed, 22 insertions(+), 86 deletions(-) diff --git a/socid_extractor/schemes.py b/socid_extractor/schemes.py index d9c863c..3f46dd5 100644 --- a/socid_extractor/schemes.py +++ b/socid_extractor/schemes.py @@ -2939,7 +2939,18 @@ def _yt_social_username(data, domain): 'Threads': { 'url_hints': ('threads.net', 'threads.com'), 'flags': ['Threads, Say more', 'barcelona'], - 'regex': r'og:title" content="(?P[^"]*?) \((?:@|@)(?P[^)]+)\)[^"]*Threads[\s\S]*?og:description" content="(?P[\d,]+) Followers[^"]*?(?P[\d,]+) Threads(?:[^"]*?(?:•|\xb7) (?:"(?P[^&]+)"|(?P[^".]*)\.))?', + 'regex': r'(?:"data":{"user":)(.*)(?:},"extensions)', + 'extract_json': True, + 'fields': { + 'uid': lambda x: x.get('id'), + 'username': lambda x: x.get('username'), + 'fullname': lambda x: x.get('full_name'), + 'bio': lambda x: x.get('biography'), + 'image': lambda x: x.get('hd_profile_pic_versions', [{}])[-1].get('url'), + 'follower_count': lambda x: x.get('follower_count'), + 'is_verified': lambda x: x.get('is_verified'), + 'links': lambda x: [entry.get('url') for entry in x.get('bio_links')], + } }, 'Smule': { 'url_hints': ('smule.com',), diff --git a/tests/test_socid_improvements.py b/tests/test_socid_improvements.py index 7ab88ed..3515409 100644 --- a/tests/test_socid_improvements.py +++ b/tests/test_socid_improvements.py @@ -1145,21 +1145,18 @@ def test_producthunt_profile_extraction(): def test_threads_profile_extraction(): - """Threads: extract fullname, username, follower/post counts, bio from OG tags.""" - html = ( - '' - '' - '' - '' - '
content
' - '' - ) + """Threads: meta tags for flags matching, info from json extracted from script tag""" + """Reference page: https://www.threads.com/@zuck""" + html = r'{"require":[["ScheduledServerJS","handle",null,[{"__bbox":{"require":[["RelayPrefetchedStreamCache","next",[],["adp_BarcelonaProfilePageDirectQueryRelayPreloader_6a6a63d9180cb2047895700",{"__bbox":{"complete":true,"result":{"data":{"user":{"profile_pic_url":"https:\/\/scontent-fco2-1.cdninstagram.com\/v\/t51.82787-19\/550174606_17925811725103224_8363667901743352243_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fco2-1.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2gEMkOHv7XIrcuKP2rmVA8iNXre2bDiMGH0Eci0gVMu8JtdfnQUNlPW3MeJsWSKawY8&_nc_ohc=vLH8jAZMCqoQ7kNvwHd8fK9&_nc_gid=PjgEQH13NsOtLYe04DBt4w&edm=APs17CUBAAAA&ccb=7-5&oh=00_AQC1i1A8NvOTFZ8seAvZw-OcQK4K9w8N1EqSO12dxh55Bg&oe=6A7024FE&_nc_sid=10d13b","friendship_status":null,"has_onboarded_to_text_post_app":true,"pk":"63055343223","text_post_app_is_private":false,"username":"zuck","text_post_app_remove_mention_entrypoint":null,"text_app_custom_feeds":null,"gating":null,"follower_count":5682725,"profile_context_facepile_users":null,"text_post_app_public_views":null,"hd_profile_pic_versions":[{"height":320,"url":"https:\/\/scontent-fco2-1.cdninstagram.com\/v\/t51.82787-19\/550174606_17925811725103224_8363667901743352243_n.jpg?stp=dst-jpg_s320x320_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fco2-1.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2gEMkOHv7XIrcuKP2rmVA8iNXre2bDiMGH0Eci0gVMu8JtdfnQUNlPW3MeJsWSKawY8&_nc_ohc=vLH8jAZMCqoQ7kNvwHd8fK9&_nc_gid=PjgEQH13NsOtLYe04DBt4w&edm=APs17CUBAAAA&ccb=7-5&oh=00_AQAINYp3OxkUL11VXhgfJZ6aK7JN-NrY9HM02Wy6kbmuXw&oe=6A7024FE&_nc_sid=10d13b","width":320},{"height":640,"url":"https:\/\/scontent-fco2-1.cdninstagram.com\/v\/t51.82787-19\/550174606_17925811725103224_8363667901743352243_n.jpg?stp=dst-jpg_s640x640_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fco2-1.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2gEMkOHv7XIrcuKP2rmVA8iNXre2bDiMGH0Eci0gVMu8JtdfnQUNlPW3MeJsWSKawY8&_nc_ohc=vLH8jAZMCqoQ7kNvwHd8fK9&_nc_gid=PjgEQH13NsOtLYe04DBt4w&edm=APs17CUBAAAA&ccb=7-5&oh=00_AQBNhNuXAjZJ55KPO-_ZpvN2UXEnXQ8Ix3kChz5OMpxwUQ&oe=6A7024FE&_nc_sid=10d13b","width":640}],"is_verified":true,"biography":"Mostly superintelligence and MMA takes","text_app_biography":{"text_fragments":{"fragments":[{"fragment_type":"plaintext","link_fragment":null,"mention_fragment":null,"plaintext":"Mostly superintelligence and MMA takes","inline_sticker_fragment":null,"linkified_web_url":null,"linkified_in_app_url":null,"styling_info":null}]}},"full_name":"Mark Zuckerberg","bio_links":[],"profile_tags":{"edges":[{"node":{"display_name":"AI Threads","name":"aithreads","id":"18399895213044171","is_community":false,"tag_cluster_name":"aithreads"}},{"node":{"display_name":"UFC Threads","name":"ufcthreads","id":"18398478472015746","is_community":false,"tag_cluster_name":"UFCThreads"}},{"node":{"display_name":"AI","name":"ai","id":"18392111818029156","is_community":false,"tag_cluster_name":"AI"}},{"node":{"display_name":"MMA","name":"mma","id":"18400625521011745","is_community":false,"tag_cluster_name":"mma"}},{"node":{"display_name":"memes","name":"memes","id":"18402364270062744","is_community":false,"tag_cluster_name":"memes"}}]},"transparency_label":null,"show_text_post_app_badge":true,"platform_podcast_info":null,"platform_podcast_episode_info":null,"id":"63055343223"}},"extensions":{"is_final":true}},"sequence_number":0}}]],["CometResourceScheduler","registerHighPriHashes",null,[["y9umSZA","1DFRiCY"]]]],"phd2_indexes":":454"}},{"__bbox":null},{"__bbox":null}]]]}' + info = extract(html) - assert info.get('fullname') == 'Mark Zuckerberg' + assert info.get('uid') == '63055343223' assert info.get('username') == 'zuck' - assert info.get('follower_count') == '12,500' - assert info.get('posts_count') == '340' - assert info.get('bio') == 'CEO of Meta' + assert info.get('fullname') == 'Mark Zuckerberg' + assert info.get('bio') == "Mostly superintelligence and MMA takes" + assert info.get('image', "").startswith('https://scontent') + assert int(info.get('follower_count', 0)) == 5682725 + assert info.get('is_verified') == 'True' # --------------------------------------------------------------------------- @@ -1634,78 +1631,6 @@ def test_taplink_nonexistent_user_not_matched(): assert not info.get('username') -def test_threads_profile(): - """Threads: extract from OG tags with HTML entities (real SPA response format).""" - html = ( - '' - '' - '' - '' - '
content
' - '' - ) - info = extract(html) - assert info.get('username') == 'fusteee' - assert info.get('fullname') == 'Marc Fusteé' - assert info.get('follower_count') == '33' - assert info.get('posts_count') == '0' - assert info.get('bio') == 'Regalame tus mejores noches' - - -def test_threads_profile_no_bio(): - """Threads: profile without bio should not capture HTML garbage.""" - html = ( - '' - '' - '' - '' - '
content
' - '' - ) - info = extract(html) - assert info.get('username') == 'alice_test' - assert info.get('fullname') == 'Alice' - assert info.get('follower_count') == '5' - assert info.get('posts_count') == '0' - assert not info.get('bio') - - -def test_threads_profile_unicode_name(): - """Threads: fullname with unicode HTML entities should be decoded.""" - html = ( - '' - '' - '' - '' - '
content
' - '' - ) - info = extract(html) - assert info.get('username') == 'bob_test' - assert info.get('fullname') == '\U0001d4d0\U0001d4fb\U0001d4fd' - assert info.get('follower_count') == '10' - assert info.get('posts_count') == '3' - assert info.get('bio') == 'hello' - - -def test_threads_profile_emoji_bio(): - """Threads: emoji bio without quotes should be extracted, not lost.""" - html = ( - '' - '' - '' - '' - '
content
' - '' - ) - info = extract(html) - assert info.get('username') == 'eve_test' - assert info.get('follower_count') == '21' - assert info.get('posts_count') == '4' - assert info.get('bio') == '🤫' - assert not info.get('bio_raw') - - def test_chess_com_html_profile(): """Chess.com HTML: extract fullname, username and image from og:meta.""" html = (