Skip to content

Commit eaa4303

Browse files
authored
fix(twitter): update package and follower response type to Profile (#638)
- Update twitterscrpaer package - Change return type from Legacy to *Profile to match twitter-scraper API - Update import to use imperatrona/twitter-scraper package - Fix type compatibility with FetchFollowers method
1 parent bf277c6 commit eaa4303

File tree

7 files changed

+50
-9
lines changed

7 files changed

+50
-9
lines changed

go.mod

+3-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ require (
1818
github.com/google/uuid v1.6.0
1919
github.com/gotd/contrib v0.20.0
2020
github.com/gotd/td v0.110.1
21+
github.com/imperatrona/twitter-scraper v0.0.15
2122
github.com/ipfs/go-cid v0.4.1
2223
github.com/ipfs/go-datastore v0.6.0
2324
github.com/ipfs/go-ds-leveldb v0.5.0
@@ -26,7 +27,7 @@ require (
2627
github.com/libp2p/go-libp2p v0.36.3
2728
github.com/libp2p/go-libp2p-kad-dht v0.26.1
2829
github.com/libp2p/go-libp2p-pubsub v0.12.0
29-
github.com/masa-finance/masa-twitter-scraper v0.0.1
30+
github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4
3031
github.com/multiformats/go-multiaddr v0.13.0
3132
github.com/multiformats/go-multihash v0.2.3
3233
github.com/onsi/ginkgo/v2 v2.20.2
@@ -42,6 +43,7 @@ require (
4243
)
4344

4445
require (
46+
github.com/AlexEidt/Vidio v1.5.1 // indirect
4547
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
4648
github.com/KyleBanks/depth v1.2.1 // indirect
4749
github.com/Microsoft/go-winio v0.6.2 // indirect

go.sum

+6-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
77
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
88
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
99
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
10+
github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE=
11+
github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI=
1012
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
1113
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
1214
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -332,6 +334,8 @@ github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei
332334
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
333335
github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
334336
github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
337+
github.com/imperatrona/twitter-scraper v0.0.15 h1:8BcaDK+pD4pPNgcV2UloHgnaIt47wCQIsaIDeGnM6rI=
338+
github.com/imperatrona/twitter-scraper v0.0.15/go.mod h1:38MY3g/h4V7Xl4HbW9lnkL8S3YiFZenBFv86hN57RG8=
335339
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
336340
github.com/ipfs/boxo v0.21.0 h1:XpGXb+TQQ0IUdYaeAxGzWjSs6ow/Lce148A/2IbRDVE=
337341
github.com/ipfs/boxo v0.21.0/go.mod h1:NmweAYeY1USOaJJxouy7DLr/Y5M8UBSsCI2KRivO+TY=
@@ -449,8 +453,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
449453
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
450454
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk=
451455
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
452-
github.com/masa-finance/masa-twitter-scraper v0.0.1 h1:n6GQZXmTCWqRr3c9m2wzKOjJfHDaRinLRmXUt10S/8U=
453-
github.com/masa-finance/masa-twitter-scraper v0.0.1/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
456+
github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4 h1:8E8Zp1raj1xIyT+zw4bp5kQJeofcmoJqo6f+4B88DCk=
457+
github.com/masa-finance/masa-twitter-scraper v0.0.0-20241126070507-4938b90b44f4/go.mod h1:Q6UBbAZsaBNDZ4hZQr8lvUUDmkJgQAhHiYWJcj0E+BM=
454458
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
455459
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
456460
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=

pkg/scrapers/twitter/cookies.go

+35-1
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,66 @@ import (
77
"os"
88
"path/filepath"
99

10-
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
10+
twitterscraper "github.com/imperatrona/twitter-scraper"
11+
12+
"github.com/sirupsen/logrus"
1113
)
1214

1315
func SaveCookies(scraper *twitterscraper.Scraper, account *TwitterAccount, baseDir string) error {
16+
logrus.Debugf("Saving cookies for user %s", account.Username)
1417
cookieFile := filepath.Join(baseDir, fmt.Sprintf("%s_twitter_cookies.json", account.Username))
1518
cookies := scraper.GetCookies()
19+
logrus.Debugf("Got %d cookies to save", len(cookies))
20+
1621
data, err := json.Marshal(cookies)
1722
if err != nil {
1823
return fmt.Errorf("error marshaling cookies: %v", err)
1924
}
25+
26+
logrus.Debugf("Writing cookies to file: %s", cookieFile)
2027
if err = os.WriteFile(cookieFile, data, 0644); err != nil {
2128
return fmt.Errorf("error saving cookies: %v", err)
2229
}
30+
logrus.Debug("Successfully saved cookies")
2331
return nil
2432
}
2533

2634
func LoadCookies(scraper *twitterscraper.Scraper, account *TwitterAccount, baseDir string) error {
35+
logrus.Debugf("Loading cookies for user %s", account.Username)
2736
cookieFile := filepath.Join(baseDir, fmt.Sprintf("%s_twitter_cookies.json", account.Username))
37+
38+
logrus.Debugf("Reading cookie file: %s", cookieFile)
2839
data, err := os.ReadFile(cookieFile)
2940
if err != nil {
3041
return fmt.Errorf("error reading cookies: %v", err)
3142
}
43+
3244
var cookies []*http.Cookie
3345
if err = json.Unmarshal(data, &cookies); err != nil {
3446
return fmt.Errorf("error unmarshaling cookies: %v", err)
3547
}
48+
logrus.Debugf("Loaded %d cookies from file", len(cookies))
49+
50+
// Verify critical cookies are present
51+
var hasAuthToken, hasCSRFToken bool
52+
for _, cookie := range cookies {
53+
if cookie.Name == "auth_token" {
54+
hasAuthToken = true
55+
logrus.Debug("Found auth_token cookie")
56+
}
57+
if cookie.Name == "ct0" {
58+
hasCSRFToken = true
59+
logrus.Debug("Found CSRF token cookie")
60+
}
61+
}
62+
63+
if !hasAuthToken || !hasCSRFToken {
64+
logrus.Debug("Missing critical authentication cookies")
65+
return fmt.Errorf("missing critical authentication cookies")
66+
}
67+
68+
logrus.Debug("Setting cookies in scraper")
3669
scraper.SetCookies(cookies)
70+
logrus.Debug("Successfully loaded and set cookies")
3771
return nil
3872
}

pkg/scrapers/twitter/followers.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ package twitter
33
import (
44
"fmt"
55

6-
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
6+
twitterscraper "github.com/imperatrona/twitter-scraper"
7+
78
"github.com/sirupsen/logrus"
89
)
910

10-
func ScrapeFollowersForProfile(baseDir string, username string, count int) ([]twitterscraper.Legacy, error) {
11+
func ScrapeFollowersForProfile(baseDir string, username string, count int) ([]*twitterscraper.Profile, error) {
1112
scraper, account, err := getAuthenticatedScraper(baseDir)
1213
if err != nil {
1314
return nil, err

pkg/scrapers/twitter/profile.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package twitter
22

33
import (
4-
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
4+
twitterscraper "github.com/imperatrona/twitter-scraper"
55
)
66

77
func ScrapeTweetsProfile(baseDir string, username string) (twitterscraper.Profile, error) {

pkg/scrapers/twitter/scraper.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package twitter
22

33
import (
4-
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
4+
twitterscraper "github.com/imperatrona/twitter-scraper"
55
)
66

77
type Scraper struct {

pkg/scrapers/twitter/tweets.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package twitter
33
import (
44
"context"
55

6-
twitterscraper "github.com/masa-finance/masa-twitter-scraper"
6+
twitterscraper "github.com/imperatrona/twitter-scraper"
77
)
88

99
type TweetResult struct {

0 commit comments

Comments
 (0)