crawl

#!/usr/bin/env python
#-*- coding: utf-8 -*-
#------------------------------------------------------
#file:crawl
#desc:控制程序,对命令行参数做处理
#author:we.agathe@gmail.com
#------------------------------------------------------
import sys,getopt
from Logger import Logger
import os
reload(sys)
sys.setdefaultencoding('utf-8')

def usage():
    print 'parameter error ...'
    print '-----------------------------'
    print '''./crawl spider 
options:
    -d path ,write output into file '''
    print '-----------------------------'

def main():
    try:
        opts,args = getopt.getopt(sys.argv[1:], 'd:')
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit()

    global log
    for k,v in opts:
        if k == '-d':
            log = Logger(v)
            sys.stdout = log
            sys.stderr = log
        elif k == '-h':
            usage()
            sys.exit()

    if len(args) != 1:
        usage()
        sys.exit()

    print args[0] + ' begining to crawl ... ...'
    try:
        m = __import__('scheduler')
        c = getattr(m, args[0])
    except:
        print 'Can not find the spider of ',args[0]
        sys.exit()
    spider = c()
    spider.start()

#--------main------------------------
if __name__ == "__main__":
    import socket
    socket.setdefaulttimeout(10)
    main()