scrapy 项目实战（一）----爬取雅昌艺术网数据

import scrapy from scrapy import Request from Demo.items import * from bs4 import BeautifulSoup import time # import sys # reload(sys) # sys.setdefaultencoding('utf-8') import re import hashlib # 加密去重def md5(str):

m = hashlib.md5() m.update(str) return m.hexdigest() #过滤注释信息，去掉换行 def replace(newline): newline = str(newline) newline = newline.replace('\r','').replace('\n','').replace('\t','').replace(' ','').replace('amp;','') re_comment = re.compile('') newlines = re_comment.sub('', newline) newlines = newlines.replace('','')

return newlines

class DemoSpider(scrapy.Spider): name = 'demo' allowed_domains = ['http://auction.artron.net/result/'] start_urls = ['http://auction.artron.net/result/pmh-0-0-2-0-1/', 'http://auction.artron.net/result/pmh-0-0-2-0-2/', 'http://auction.artron.net/result/pmh-0-0-2-0-4/', 'http://auction.artron.net/result/pmh-0-0-2-0-5/', 'http://auction.artron.net/result/pmh-0-0-2-0-6/', 'http://auction.artron.net/result/pmh-0-0-2-0-7/', 'http://auction.artron.net/result/pmh-0-0-2-0-8/', 'http://auction.artron.net/result/pmh-0-0-2-0-9/', 'http://auction.artron.net/result/pmh-0-0-2-0-10/', 'http://auction.artron.net/result/pmh-0-0-2-0-3/']

def parse(self, response): html = response.text soup = BeautifulSoup(html,'html.parser') result_lists = soup.find_all('ul',attrs={"class":"dataList"})[0] result_lists_replace = replace(result_lists) result_lists_replace = result_lists_replace.decode('utf-8') result_list = re.findall('

• (.*?)

session_address = session_address_time[0] item_auct_time = session_address_time[1]

hashcode = md5(str(auction_url)) create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))

import scrapy

# Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import json import MySQLdb

def insert_data(dbName,data_dict):

try:

data_values = "(" + "%s," * (len(data_dict)) + ")" data_values = data_values.replace(',)', ')')

dbField = data_dict.keys() dataTuple = tuple(data_dict.values()) dbField = str(tuple(dbField)).replace("'",'') conn = MySQLdb.connect(host="10.10.10.77", user="xuchunlin", passwd="ed35sdef456", db="epai_spider_2018", charset="utf8") cursor = conn.cursor() sql = """ insert into %s %s values %s """ % (dbName,dbField,data_values) params = dataTuple cursor.execute(sql, params) conn.commit() cursor.close() conn.close()

print "===== 插入成功 =====" return 1

except Exception as e: print "******** 插入失败 ********" print e return 0

class DemoPipeline(object):

# Scrapy settings for Demo project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'Demo'

SPIDER_MODULES = ['Demo.spiders'] NEWSPIDER_MODULE = 'Demo.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'Demo (+http://www.yourdomain.com)'

# Obey robots.txt rules ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default) #COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False

# Override the default request headers:

DEFAULT_REQUEST_HEADERS = {

"Host":"auction.artron.net", # "Connection":"keep-alive", # "Upgrade-Insecure-Requests":"1", "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Referer":"http://auction.artron.net/result/pmh-0-0-2-0-2/", "Accept-Encoding":"gzip, deflate", "Accept-Language":"zh-CN,zh;q=0.8", "Cookie":"td_cookie=2322469817; gr_user_id=84f865e6-466f-4386-acfb-e524e8452c87; gr_session_id_276fdc71b3c353173f111df9361be1bb=ee1eb94e-b7a9-4521-8409-439ec1958b6c; gr_session_id_276fdc71b3c353173f111df9361be1bb_ee1eb94e-b7a9-4521-8409-439ec1958b6c=true; _at_pt_0_=2351147; _at_pt_1_=A%E8%AE%B8%E6%98%A5%E6%9E%97; _at_pt_2_=e642b85a3cf8319a81f48ef8cc403d3b; Hm_lvt_851619594aa1d1fb8c108cde832cc127=1533086287,1533100514,1533280555,1534225608; Hm_lpvt_851619594aa1d1fb8c108cde832cc127=1534298942",

}

# Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # 'Demo.middlewares.DemoSpiderMiddleware': 543, #}

# Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # 'Demo.middlewares.MyCustomDownloaderMiddleware': 543, #}

# Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #}

# Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html

ITEM_PIPELINES = { 'Demo.pipelines.DemoPipeline': 300, }

# Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False