Py - 全球主机交流论坛 - Powered by Discuz!

import requests
from bs4 import BeautifulSoup
import os
import time
class Spider:
def __init__(self, base_url):
self.base_url = base_url
def run(self):
for i in range(1, 10000):
url = '{}/{}'.format(base_url, i)
html = self.request(url)
if html:
self.parse_html(html, url)
time.sleep(1)
else:
time.sleep(0.5)
def parse_html(self, html, href):
title = BeautifulSoup(html.text, 'lxml').find('h2', class_='main-title').text
print('开始保存：{}'.format(title))
path = str(title)
self.mkdir(path)
max_span = BeautifulSoup(html.text, 'lxml').find('div', class_='pagenavi').find_all('span')[-2].text
for page in range(1, int(max_span) + 1):
if page == 1:
page_url = href
else:
page_url = '{}/{}'.format(href, str(page))
self.parse_img(page_url, img_name=page)
def parse_img(self, page_url, img_name):
img_html = self.request(page_url)
img_url = BeautifulSoup(img_html.text, 'lxml').find('div', class_='main-image').find('img')['src']
self.save_img(img_url, img_name)
def save_img(self, img_url, img_name):
img = self.request(img_url)
with open('{}.jpg'.format(img_name), 'ab') as f:
f.write(img.content)
def request(self, url):
headers = {
'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64)"
" AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"
}
content = requests.get(url, headers=headers, allow_redirects=False)
if content.status_code != 200:
return False
else:
return content
def mkdir(self, path): # 这个函数创建文件夹
path = path.strip()
isExists = os.path.exists(os.path.join("D:\mzitu", path))
if not isExists:
print(u'创建', path, u'文件夹')
os.makedirs(os.path.join("D:\MZITU", path))
os.chdir(os.path.join("D:\mzitu", path)) # 切换到目录
return True
else:
print(u'名字叫做', path, u'的文件夹已经存在了')
return False
if __name__ == '__main__':
base_url = 'http://www.mzitu.com'
spider = Spider(base_url)
spider.run()

复制代码