CFP crawler

cfp_crawler/.gitignore

@@ -0,0 +1,5 @@
+
+*.pyc
+*.log
+
+

cfp_crawler/README.md

@@ -0,0 +1,3 @@
+### Basic Usage
+
+scrapy crawl crawler

cfp_crawler/proposal/items.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class ProposalItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

cfp_crawler/proposal/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class ProposalSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class ProposalDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

cfp_crawler/proposal/pipelines.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+class ProposalPipeline(object):
+    def process_item(self, item, spider):
+        return item

cfp_crawler/proposal/settings.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for proposal project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'proposal'
+
+SPIDER_MODULES = ['proposal.spiders']
+NEWSPIDER_MODULE = 'proposal.spiders'
+LOG_FILE = "logs.log"
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'proposal (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'proposal.middlewares.ProposalSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'proposal.middlewares.ProposalDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'proposal.pipelines.ProposalPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

cfp_crawler/proposal/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

cfp_crawler/proposal/spiders/crawler.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+from scrapy import signals
+
+
+class CrawlerSpider(scrapy.Spider):
+    name = 'crawler'
+    allowed_domains = ['in.pycon.org']
+    url = "https://in.pycon.org"
+    proposals = []
+    file = open("proposals.json", "w")
+    def start_requests(self):
+        yield scrapy.Request(self.url + "/cfp/2018/proposals", callback = self.parse)
+
+    def parse(self, response):
+        proposal_links = response.xpath("//h3[@class='proposal--title']/a/@href").extract()
+        for link in proposal_links:
+            yield scrapy.Request(self.url + link, callback = self.parseProposal)
+
+
+
+    def parseProposal(self, response):
+        title = response.xpath("//h1[@class='proposal-title']/text()").extract()[0].strip()
+        author = response.xpath("//p[@class='text-muted']/small/b/text()").extract()[0].strip()
+        created_on = response.xpath("//p[@class='text-muted']/small/b/time/text()").extract()[0].strip()
+
+        section = response.xpath("//section[@class='col-sm-8 proposal-writeup']/div")
+        proposal = {}
+        for div in section:
+            heading = div.xpath(".//h4[@class='heading']/b/text()").extract()[0]
+            data = self.format_data(div.xpath(".//text()").extract(), heading)
+            data = data[2:-2]
+            proposal[heading[:-1]] = data
+
+        table_rows = response.xpath("//table/tr")
+        for row in table_rows:
+            extra_info_heading = row.xpath(".//td/small/text()").extract()[0].strip()
+            extra_info_content = row.xpath(".//td/text()").extract()[0].strip()
+            proposal[extra_info_heading[:-1]] = extra_info_content
+
+        proposal["title"] = title
+        proposal["link_to_proposal"] = response.request.url
+        proposal["author"] = author
+        proposal["created_on"] = created_on
+        proposal["Last Updated"] = response.xpath("//time/text()").extract()[0]
+
+        self.proposals.append(proposal)
+
+    def format_data(self, data, head):
+        return " ".join([d.strip() for d in data if d != "" and d!=head ])
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super(CrawlerSpider, cls).from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider.spider_closed, signals.spider_closed)
+        return spider
+
+    def spider_closed(self, spider):
+        print("Closing spider")
+        json.dump(self.proposals, self.file, indent = 2, sort_keys = True)
+
+
+

cfp_crawler/proposals.json

@@ -0,0 +1,16 @@
+[
+  {
+    "Description": "Vyper is a recently launched python based smart contract programming language. The talk will focus on the features and benefits of Vyper and compare it to Solidity which is similar to Javascript and will include brief demos comparing smart contract implementations.\nTopics to be covered:   Features of Vyper and their comparisons to solidity  Design pattern of smart contracts  Creating smart contracts : demos in both languages", 
+    "Last Updated": "10 Jul, 2018", 
+    "Prerequisites": "Basic knowledge about Blockchain and the Ethereum ecosystem would be helpful", 
+    "Section": "Others", 
+    "Speaker Info": "The speaker is currently working as a research associate at IIIT Delhi and has worked on the Ethereum blockchain  as smart contract developer  building decentralised applications and web3js based frontends for these applications", 
+    "Speaker Links": "You can reach me at :  https://aerophile.github.io  https://twitter.com/shubham0075_", 
+    "Target Audience": "Intermediate", 
+    "Type": "Talks", 
+    "author": "Shubham Gupta (~shubham98)", 
+    "created_on": "10 Jul, 2018", 
+    "link_to_proposal": "https://in.pycon.org/cfp/2018/proposals/vyper-vs-solidity-smart-contracts-in-the-python-ecosystem~bok3b/", 
+    "title": "Vyper vs Solidity: Smart contracts in the Python ecosystem"
+  }
+]

cfp_crawler/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = proposal.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = proposal