Scraper for basic information #1
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,18 @@ | ||
| { | ||
| "actorSpecification": 1, | ||
| "name": "factorio-scraper", | ||
| "title": "Factorio forums scraper", | ||
| "description": "Scraper for factorio forums that extracts topics and their details.", | ||
| "version": "0.1", | ||
| "buildTag": "latest", | ||
| "meta": { | ||
| "templateId": "ts-empty", | ||
| "generatedBy": "GitHub Copilot with Claude-Sonnet-4.5" | ||
| }, | ||
| "dockerfile": "../Dockerfile", | ||
| "inputSchema": "./inputSchema.json", | ||
| "outputSchema": "./outputSchema.json", | ||
| "storages": { | ||
| "dataset": "./datasetSchema.json" | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| { | ||
| "actorSpecification": 1, | ||
| "title": "Factorio topics dataset schema", | ||
| "description": "Schema for Factorio forum topics dataset", | ||
| "fields": { | ||
| "type": "object", | ||
| "additionalProperties": false, | ||
| "properties": { | ||
| "publishedAt": { | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "author": { | ||
| "type": "string" | ||
| }, | ||
| "text": { | ||
| "type": "string" | ||
| }, | ||
| "topic": { | ||
| "type": "object", | ||
| "additionalProperties": false, | ||
| "properties": { | ||
| "category": { | ||
| "type": "string" | ||
| }, | ||
| "title": { | ||
| "type": "string" | ||
| }, | ||
| "url": { | ||
| "type": "string", | ||
| "format": "uri" | ||
| }, | ||
| "author": { | ||
| "type": "string" | ||
| }, | ||
| "createdAt": { | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "lastPostAt": { | ||
| "type": "string", | ||
| "format": "date-time" | ||
| }, | ||
| "views": { | ||
| "type": "number" | ||
| }, | ||
| "isAnnouncement": { | ||
| "type": "boolean" | ||
| }, | ||
| "totalPosts": { | ||
| "type": "integer" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "views": { | ||
| "overview": { | ||
| "title": "Overview", | ||
| "transformation": { | ||
| "fields": [ | ||
| "topic.category", | ||
| "topic.title", | ||
| "topic.url", | ||
| "topic.author", | ||
| "topic.createdAt", | ||
| "topic.lastPostAt", | ||
| "topic.views", | ||
| "topic.isAnnouncement", | ||
| "topic.totalPosts", | ||
| "author", | ||
| "publishedAt", | ||
| "text" | ||
| ] | ||
| }, | ||
| "display": { | ||
| "component": "table", | ||
| "properties": { | ||
| "topic.category": { | ||
| "label": "Category", | ||
| "format": "text" | ||
| }, | ||
| "topic.title": { | ||
| "label": "Topic Title", | ||
| "format": "text" | ||
| }, | ||
| "topic.url": { | ||
| "label": "Topic URL", | ||
| "format": "link" | ||
| }, | ||
| "topic.author": { | ||
| "label": "Topic Author", | ||
| "format": "text" | ||
| }, | ||
| "topic.createdAt": { | ||
| "label": "Topic Created At", | ||
| "format": "date" | ||
| }, | ||
| "topic.lastPostAt": { | ||
| "label": "Last Post At", | ||
| "format": "date" | ||
| }, | ||
| "topic.views": { | ||
| "label": "Views", | ||
| "format": "number" | ||
| }, | ||
| "topic.isAnnouncement": { | ||
| "label": "Announcement", | ||
| "format": "boolean" | ||
| }, | ||
| "topic.totalPosts": { | ||
| "label": "Total Posts", | ||
| "format": "number" | ||
| }, | ||
| "author": { | ||
| "label": "Post Author", | ||
| "format": "text" | ||
| }, | ||
| "publishedAt": { | ||
| "label": "Post Published At", | ||
| "format": "date" | ||
| }, | ||
| "text": { | ||
| "label": "Post Text", | ||
| "format": "text" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| { | ||
| "title": "Factorio Scraper Input Schema", | ||
| "type": "object", | ||
| "schemaVersion": 1, | ||
| "properties": { | ||
| "category": { | ||
| "title": "Category", | ||
| "type": "string", | ||
| "description": "The category to scrape, case insensitive", | ||
| "editor": "textfield", | ||
| "prefill": "news" | ||
| }, | ||
| "limit": { | ||
| "title": "Limit", | ||
| "type": "number", | ||
| "description": "The maximum number of topics to scrape, there are 25 topics per page", | ||
| "editor": "number", | ||
| "prefill": 100, | ||
| "minimum": 1 | ||
| } | ||
| }, | ||
| "required": ["category"] | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "actorOutputSchemaVersion": 1, | ||
| "description": "Output schema for factorio scraper", | ||
| "title": "Output schema for factorio scraper", | ||
| "properties": { | ||
| "dataset": { | ||
| "type": "string", | ||
| "title": "Dataset", | ||
| "template": "{{links.apiDefaultDatasetUrl}}/items" | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,67 +1,40 @@ | ||
| This scraper collects data from forum of the game [Factorio](https://forums.factorio.com/). | ||
| It does not go around the login, so it does not scrape user detail page. | ||
|
|
||
| It scrapes only one category from the forum which is provided as input. | ||
| There is also option to scrape only specific number of topics. | ||
| A topic is a page with posts provide by users. | ||
|
|
||
| ## Inputs | ||
|
|
||
| For up to date information about inputs see the [input page](https://console.apify.com/actors/A448q9dopTHFBMkUf/input) | ||
|
|
||
| | Name | Required | Use | | ||
| | -------- | -------- | ------------------------------------ | | ||
| | Category | yes | Category to scrape from the page | | ||
| | Limit | yes | How many topics to scrape at maximum | | ||
|
|
||
| ## Output | ||
|
|
||
| The output is a single comment (post) on the topic with all associated information about the topic this comment belongs to. | ||
|
|
||
| ### Example output | ||
|
|
||
| ``` | ||
| { | ||
| "author": "Carl", | ||
| "publishedAt": "2020-09-07T12:30:00+00:00", | ||
| "text": "This game is the best!", | ||
| "topic": { | ||
| "category": "news", | ||
| "title": "Is this the best game?", | ||
| "views": 12345, | ||
| "url": "https://forums.factorio.com/viewtopic.php?t=0000001", | ||
| "author": "FactorioBot", | ||
| "createdAt": "2020-01-01T12:00:01+00:00", | ||
| "lastPostAt": "2020-02-01T23:13:37+00:00", | ||
| "isAnnouncement": true, | ||
| "totalPosts": 123 | ||
| } | ||
| } | ||
| ``` | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| export const BASE_URL = 'https://forums.factorio.com'; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,33 @@ | ||
| import { Actor } from 'apify'; | ||
| import { CheerioCrawler, Request } from 'crawlee'; | ||
|
|
||
| import { BASE_URL } from './constants.js'; | ||
| import { router } from './routes.js'; | ||
| import { type Input, Label, type UserData } from './types/index.js'; | ||
|
|
||
| await Actor.init(); | ||
|
|
||
| const { category, limit } = (await Actor.getInput<Input>())!; | ||
|
|
||
| const proxyConfiguration = await Actor.createProxyConfiguration({ | ||
| useApifyProxy: true, | ||
| }); | ||
|
|
||
| const crawler = new CheerioCrawler({ | ||
| proxyConfiguration, | ||
| requestHandler: router, | ||
| // this is 2 requests per second, this should be fine and not overload the site since it is not so frequently used one | ||
| maxRequestsPerMinute: 120, | ||
|
Collaborator
There was a problem hiding this comment. A bit of issue with this option is that (I think) it doesn't distribute the requests in the minute, it will just stop processing once done. So it could still lead to high bursts at the start (not sure how likely it is in practice). You could solve that by adding maxConcurrency limit as well.
Owner
Author
There was a problem hiding this comment. yes I noticed that it just does huge burst at the beginning and then it slows down. Will try to do this with the |
||
| }); | ||
|
|
||
| const initialRequest = new Request<UserData[typeof Label.FORUM_SEARCH_START]>({ | ||
| url: `${BASE_URL}`, | ||
| label: Label.FORUM_SEARCH_START, | ||
| userData: { | ||
| category, | ||
| limit, | ||
| }, | ||
| }); | ||
| await crawler.run([initialRequest]); | ||
|
|
||
| await Actor.exit(); | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| import { createCheerioRouter } from 'crawlee'; | ||
|
|
||
| import { forumCategoryHandler } from './routes/forumCategory.js'; | ||
| import { forumSearchStartHandler } from './routes/searchStart.js'; | ||
| import { topicDetailHandler } from './routes/topicDetail.js'; | ||
| import { Label } from './types/label.js'; | ||
|
|
||
| export const router = createCheerioRouter(); | ||
|
|
||
| router.addHandler(Label.FORUM_SEARCH_START, forumSearchStartHandler); | ||
| router.addHandler(Label.FORUM_CATEGORY, forumCategoryHandler); | ||
| router.addHandler(Label.TOPIC_DETAIL, topicDetailHandler); |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We generally prefer to link the Input page to the Actor Store page, which is derived from the inpjt schema. That way there is only single source of truth for the descriptions, you will often forget to change the readme.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I linked the input page statically in the README.