抓取美女图片的爬虫小程序 - Crossin的编程教室

#!/usr/bin/env python
#coding:UTF-8
import urllib
import re
import os
import os.path
index=0
#抓取页面的函数
def getPage(url):
page=urllib.urlopen(url).read()
return page
#抓取首页美女分类的链接信息
def getSortLinkInfo(html):
patt='<a href="/[^\s]+?\.html" title="[^\s]+?"'
regex=re.compile(patt)
linkInfo=re.findall(regex,html)
links={}
for i in linkInfo:
#links.append('http://www.22mm.cc'+i.split('"')[1])
links['http://www.22mm.cc'+i.split('"')[1]]=i.split('"')[3]
return links #links是有效链接的列表
#获取特定美女页面中的链接信息
def getBeautyLinkInfo(link):
page=getPage(link)
patt='<a href=\'[^\s]+?-\d+?\.html\'>\d+?</a>'
regex=re.compile(patt)
lastLink=re.findall(regex,page)
if len(lastLink)>0:
lastLink=lastLink[-1].split("'")[1]
return lastLink #lastLink是最后一个美女页面的相对路径
#提取最终美女图片的链接
def getImgLinks(lastLink):
page=getPage(lastLink)
patt='arrayImg\[0\]="(http://[^\s]+?\.jpg)"'
regex=re.compile(patt)
imgLinks=re.findall(regex,page)
return imgLinks #imgLinks是有效的图片链接的列表
#下载并且保存图片
def saveImg(imgLinks,dirname):
global index
path=unicode('D:\\pics\\'+dirname,'utf8')
os.mkdir('%s' %(path))
dirname=dirname.decode('utf8')
for i in imgLinks:
urllib.urlretrieve(i,'D:\\pics\%s\%d.jpg' % (dirname,index))
print '%s has been downloaded and saved successfully.'%(i)
index+=1
indexURL='http://www.22mm.cc'
def start():
homePage=getPage(indexURL)
links=getSortLinkInfo(homePage)
for i in links:
#dirname=unicode('D:\\pics\\'+links[i],'utf8')
#os.mkdir('%s' %(dirname))
dirname=links[i]
relPath=getBeautyLinkInfo(i)
if len(relPath)>0:
lastLink='http://www.22mm.cc/mm/'+i.split("/")[4]+'/'+relPath
tempLinks=getImgLinks(lastLink)
imgLinks=[]
for j in tempLinks:
imgLinks.append(re.sub('big','pic',j))
saveImg(imgLinks,dirname)
start()

复制代码

#!usr/bin/env python
#coding:UTF-8
import urllib2
import urllib
import re
import os
# 获取页面的html
def get_page(url):
req=urllib2.Request(url)
try:
html=urllib2.urlopen(req).read()
return html
except urllib2.URLError,e:
if e.code==404:
return False
#获取美女图片的四个分类链接
def get_fen_lei_link(html):
patt='<a href="/mm/[^\s]+?/" >'
regex=re.compile(patt)
fen_lei_link=[]
temp_link=re.findall(regex,html)[0:4]
for link in temp_link:
fen_lei_link.append('http://www.22mm.cc'+link.split('"')[1])
return fen_lei_link
#获取套图的链接
def get_taotu_link(specific_page):
global taotu_links
taotu_links={}
patt='<a href="/[^\s]+?\.html" title=".+?"'
regex=re.compile(patt)
link_info=re.findall(regex,specific_page)
for i in link_info:
taotu_links['http://www.22mm.cc'+i.split('"')[1]]=i.split('"')[3]
#获取套图页面中指向最后一个图片的链接
def get_taotu_last_link(taotu_link):
taotu_page=get_page(taotu_link)
patt='<a href=\'[^\s]+?-\d+?\.html\'>\d+?</a>'
regex=re.compile(patt)
taotu_last_link=re.findall(regex,taotu_page)[-1].split("'")[1]
return taotu_last_link
#获取暂时的图片链接
def get_temp_image_link(taotu_last_link):
taotu_last_page=get_page(taotu_last_link)
patt=patt='arrayImg\[\d\]="(http://[^\s]+?\.jpg)"'
regex=re.compile(patt)
temp_image_link=re.findall(regex,taotu_last_page)
return temp_image_link
#将图片下载并且保存到D盘的pic文件夹中
def save_image(image_links,dirname):
global index
path=unicode('D:\\pic\\'+dirname,'utf8')
os.mkdir('%s' %(path))
dirname=dirname.decode('utf8')
for img_link in image_links:
urllib.urlretrieve(img_link,'D:\\pic\%s\%d.jpg' % (dirname,index))
print '%s has been downloaded and saved successfully.'%(img_link)
index+=1
def start():
print 'Waiting............'
url='http://www.22mm.cc'
html=get_page(url)
fen_lei_link=get_fen_lei_link(html)
for link in fen_lei_link:
temp_link=link
page_index=1 #page_index是页面索引
specific_page=get_page(link)
get_taotu_link(specific_page)
while page_index<4: #套图的数目很多，暂时只抓取每个分类的前三页图片
if page_index==1:
specific_page=get_page(link)
get_taotu_link(specific_page)
else:
link=temp_link+'index_%d.html' %(page_index)
specific_page=get_page(link)
if not specific_page:
break
else:
get_taotu_link(specific_page)
for key in taotu_links:
dirname=taotu_links[key]
temp_taotu_last_link=get_taotu_last_link(key)
taotu_last_link='http://www.22mm.cc/mm/'+key.split("/")[4]+'/'+temp_taotu_last_link
temp_image_link=get_temp_image_link(taotu_last_link)
image_links=[]
for each in temp_image_link:
image_links.append(re.sub('big','pic',each))
save_image(image_links,dirname)
page_index+=1
#links是存储套图链接信息的字典
taotu_links={}
#index是图片名称的索引
index=0
start()

复制代码

from bs4 import BeautifulSoup
import urllib.request
from collections import deque,defaultdict
import re
import os
url='http://www.22mm.cc/'
def get_page(url):
headers = {
'Connection': 'Keep-Alive',
'Accept': "image/png,image/*;q=0.8,*/*;q=0.5",
'Accept-Language': "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
'Referer':"http://www.22mm.cc/"
}
req = urllib.request.Request(url, headers=headers)
response = urllib.request.urlopen(req)
responseutf8 = response.read().decode()
responsegbk = responseutf8.encode('gbk', 'ignore')
page = responsegbk.decode('gbk')
return page
def get_first_link(page):
queue=deque()
filename=[]
baseurl='http://22mm.xiuna.com'
soup=BeautifulSoup(page)
pic=soup.find(id = 'recshowBox')
for child in pic.children:
queue.append(baseurl+child['href'])
filename.append(child['title'])
return filename,queue
def get_all_links(filename, queue):
d=defaultdict(set)
n=0
while queue:
link2=[]
linkss=link2[:]
links=link2[:]
link1=link2[:]
baselink=queue.popleft()
pagecode=get_page(baselink)
soup=BeautifulSoup(pagecode)
link=soup.find(class_="pagelist")
for child in link.children:
try:
linkss.append(child['href'])
except:
continue
links=linkss[1:-1]
pattern=re.compile('http.*/')
addurl=re.findall(pattern,baselink)
link1=[addurl[0]+link for link in links]
'''for link in links:
link1.append(addurl[0]+link)
link1.append(baselink)'''
for value in link1:
d[filename[n]].add(value)
n+=1
return d
def download_all_pic(d):
for key in d:
print("正在创建{}的目录".format(key))
path='D:\\pics\\'+key+'\\'
os.mkdir('%s'%path)
print(path)
print("开始下载{}的图集...".format(key))
index=1
for i in d[key]:
global index
page1=get_page(i)
pattern1=re.compile(r'arrayImg\[0\]="(http.*?jpg)')
addurl1=re.findall(pattern1,page1)
down=addurl1[-1].replace('big','pic')
print("正在下载%s的第%d张图片" % (key,index))
urllib.request.urlretrieve(down,'D:\\pics\%s\%d.jpg' % (key,index))
print("下载完成")
index+=1
def start(url):
page=get_page(url)
filename, queue=get_first_link(page)
d=get_all_links(filename,queue)
download_all_pic(d)
start(url)

复制代码

#获取美女图片的四个分类链接
def get_fen_lei_link(html):
patt='<a href="/mm/[^\s]+?/" >'
regex=re.compile(patt)
fen_lei_link=[]
temp_link=re.findall(regex,html)[0:4]
for link in temp_link:
fen_lei_link.append('http://www.22mm.cc'+link.split('"')[1])
return fen_lei_link

复制代码