添加 'main.py'
This commit is contained in:
parent
4b47204e08
commit
449cdb33ad
211
main.py
Normal file
211
main.py
Normal file
@ -0,0 +1,211 @@
|
|||||||
|
import requests
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
from lxml import etree
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from queue import Queue
|
||||||
|
|
||||||
|
'''
|
||||||
|
Time:2020年6月16日18:40:20
|
||||||
|
By:Eg.
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
爬取weibo某话题下评论中的图片
|
||||||
|
https://m.weibo.cn/status/4489758501815678?
|
||||||
|
https://weibo.com/6524978930/IF9XHxJpF?type=repost#_rnd1588381011696
|
||||||
|
'''
|
||||||
|
|
||||||
|
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'}
|
||||||
|
cookies = ''
|
||||||
|
#取消掉证书warning
|
||||||
|
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
|
||||||
|
|
||||||
|
def visitorlogin():
|
||||||
|
'''
|
||||||
|
visitorlogin
|
||||||
|
return cookies
|
||||||
|
'''
|
||||||
|
|
||||||
|
#-------游客登录1
|
||||||
|
url = 'https://passport.weibo.com/visitor/genvisitor'
|
||||||
|
data = {'cb':'gen_callback',
|
||||||
|
'fp':json.dumps({"os":"1",
|
||||||
|
"browser":"Chrome81,0,4044,129",
|
||||||
|
"fonts":"undefined",
|
||||||
|
"screenInfo":"1280*800*24",
|
||||||
|
"plugins":"Portable Document Format::internal-pdf-viewer::Chrome PDF Plugin|::mhjfbmdgcfjbbpaeojofohoefgiehjai::Chrome PDF Viewer|::internal-nacl-plugin::Native Client"
|
||||||
|
})
|
||||||
|
}
|
||||||
|
r = requests.post(url,data=data,headers=headers,verify=False)
|
||||||
|
text = re.findall('gen_callback\((.*?)\);',r.text)[0]
|
||||||
|
tid = json.loads(text)['data']['tid']
|
||||||
|
#-------游客登录2
|
||||||
|
params = {'a':'incarnate',
|
||||||
|
't':tid,
|
||||||
|
'cb':'cross_domain'
|
||||||
|
}
|
||||||
|
url = 'https://passport.weibo.com/visitor/visitor'
|
||||||
|
r = requests.get(url,params=params,headers=headers,verify=False)
|
||||||
|
text = re.findall('cross_domain\((.*?)\);',r.text)[0]
|
||||||
|
sub = json.loads(text)['data']['sub']
|
||||||
|
subp = json.loads(text)['data']['subp']
|
||||||
|
#-------游客登录3
|
||||||
|
url = 'https://login.sina.com.cn/visitor/visitor'
|
||||||
|
params = {'a':'crossdomain',
|
||||||
|
's':sub,
|
||||||
|
'sp':subp,
|
||||||
|
'cb':'return_back'
|
||||||
|
}
|
||||||
|
r = requests.get(url,params=params,headers=headers,verify=False)
|
||||||
|
return r.cookies.get_dict()
|
||||||
|
|
||||||
|
def base62_decode(string):
|
||||||
|
'''
|
||||||
|
来自万能的百度
|
||||||
|
'''
|
||||||
|
alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||||
|
base = len(alphabet)
|
||||||
|
strlen = len(string)
|
||||||
|
num = 0
|
||||||
|
|
||||||
|
idx = 0
|
||||||
|
for char in string:
|
||||||
|
power = (strlen - (idx + 1))
|
||||||
|
num += alphabet.index(char) * (base ** power)
|
||||||
|
idx += 1
|
||||||
|
|
||||||
|
return num
|
||||||
|
|
||||||
|
def mid_to_url(midint):
|
||||||
|
'''
|
||||||
|
来自万能的百度
|
||||||
|
'''
|
||||||
|
|
||||||
|
url = midint
|
||||||
|
url = str(url)[::-1]
|
||||||
|
size = len(url) / 4 if len(url) % 4 == 0 else len(url) / 4 + 1
|
||||||
|
result = []
|
||||||
|
for i in range(int(size)):
|
||||||
|
s = url[i * 4: (i + 1) * 4][::-1]
|
||||||
|
s = str(base62_decode(str(s)))
|
||||||
|
s_len = len(s)
|
||||||
|
if i < size - 1 and s_len < 7:
|
||||||
|
s = (7 - s_len) * '0' + s
|
||||||
|
result.append(s)
|
||||||
|
result.reverse()
|
||||||
|
return int(''.join(result))
|
||||||
|
|
||||||
|
|
||||||
|
class Th(threading.Thread):
|
||||||
|
def __init__(self, queue):
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
self.__queue = queue
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while True:
|
||||||
|
test = self.__queue.get()
|
||||||
|
saveimg(test[0],test[1],test[2])
|
||||||
|
self.__queue.task_done()
|
||||||
|
|
||||||
|
|
||||||
|
queue = Queue(20)#线程数
|
||||||
|
for i in range(queue.maxsize):
|
||||||
|
t = Th(queue)
|
||||||
|
t.setDaemon(True)
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
def saveimg(weiboid,comment_id,img_url):
|
||||||
|
#print(img_url)
|
||||||
|
filename = '{}/{}{}'.format(weiboid,comment_id,os.path.basename(img_url))
|
||||||
|
if os.path.exists(filename) == False:
|
||||||
|
with open(filename,'wb') as f:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
r_img = requests.get(img_url,headers=headers,cookies=cookies,timeout=5)
|
||||||
|
except:
|
||||||
|
print('下载pic出错',img_url)
|
||||||
|
else:
|
||||||
|
if r_img.status_code == 200:
|
||||||
|
break
|
||||||
|
f.write(r_img.content)
|
||||||
|
|
||||||
|
|
||||||
|
def getweiboid(string):
|
||||||
|
'''
|
||||||
|
https://m.weibo.cn/status/4489758501815678? 这种直接出4489758501815678
|
||||||
|
https://m.weibo.cn/status/J5PjY66K0?
|
||||||
|
https://weibo.com/6524978930/IF9XHxJpF?type=repost#_rnd1588381011696 这种需要转换IF9XHxJpF
|
||||||
|
'''
|
||||||
|
|
||||||
|
if 'm.weibo' in string:
|
||||||
|
#目测都是16位
|
||||||
|
weiboid = re.findall('/(\d{16})',string)[0]
|
||||||
|
else:
|
||||||
|
weiboid = re.findall('/\d+/([A-Za-z0-9]*)',string)[0]
|
||||||
|
weiboid = mid_to_url(weiboid)
|
||||||
|
return weiboid
|
||||||
|
|
||||||
|
def spider(weibourl):
|
||||||
|
global cookies
|
||||||
|
|
||||||
|
weiboid = getweiboid(weibourl)
|
||||||
|
#时间排列api
|
||||||
|
url = 'https://weibo.com/aj/v6/comment/big?ajwvr=6&id={}&filter=all&from=singleWeiBo'.format(weiboid)
|
||||||
|
if os.path.exists(weiboid) == False:
|
||||||
|
os.makedirs(weiboid)
|
||||||
|
#循环获取每一页
|
||||||
|
while True:
|
||||||
|
print(url)
|
||||||
|
tt = True
|
||||||
|
retry = 0
|
||||||
|
#重试专用
|
||||||
|
while tt:
|
||||||
|
r = requests.get(url,headers=headers,cookies=cookies)
|
||||||
|
html = etree.HTML(json.loads(r.text)['data']['html'])
|
||||||
|
#单个评论div
|
||||||
|
for div in html.xpath('/html/body/div/div/div[@class="list_li S_line1 clearfix"]'):
|
||||||
|
comment_id = div.xpath('./@comment_id')[0]
|
||||||
|
#通常用这种来展示
|
||||||
|
for imgurl in div.xpath('./div/div/div/ul/li/img/@src'):
|
||||||
|
#替换thumb180字段来获取高清大图
|
||||||
|
img_url = 'https:' + imgurl.replace('thumb180','large')
|
||||||
|
print('piclink:',img_url)
|
||||||
|
#saveimg(weiboid,comment_id,img_url)
|
||||||
|
queue.put([weiboid,comment_id,img_url])
|
||||||
|
|
||||||
|
#有时候weibo会以这种方式展示,代表请求太快
|
||||||
|
#for imgurl in div.xpath('./div/div/a[@title="网页链接"]/@href'):
|
||||||
|
if len(div.xpath('./div/div/a[@title="网页链接"]/@href')) ==0:
|
||||||
|
tt = False
|
||||||
|
else:
|
||||||
|
cookies = visitorlogin()
|
||||||
|
retry = retry + 1
|
||||||
|
print('retry',retry)
|
||||||
|
|
||||||
|
|
||||||
|
#获取下一页地址
|
||||||
|
baseurl = 'https://weibo.com/aj/v6/comment/big?{}&from=singleWeiBo'
|
||||||
|
#'&from=singleWeiBo'不能去掉，不然会出现各种balabala问题
|
||||||
|
try:
|
||||||
|
url = baseurl.format(html.xpath('/html/body/div/div/div[@node-type="comment_loading"]/@action-data')[0])
|
||||||
|
except IndexError:
|
||||||
|
try:
|
||||||
|
url = baseurl.format(html.xpath('/html/body/div/div/div/div/a[@class="page next S_txt1 S_line1"]/span/@action-data')[0])
|
||||||
|
except IndexError:
|
||||||
|
try:
|
||||||
|
url = baseurl.format(html.xpath('/html/body/div/div/a[@action-type="click_more_comment"]/@action-data')[0])
|
||||||
|
except IndexError:
|
||||||
|
## with open('test.html','w',encoding='utf-8') as f:
|
||||||
|
## f.write(json.loads(r.text)['data']['html'])
|
||||||
|
break #没意外的话应该是已经获取完
|
||||||
|
cookies = visitorlogin()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
cookies = visitorlogin()
|
||||||
|
print('cookies:',cookies)
|
||||||
|
spider('https://m.weibo.cn/detail/4511863448505629?')
|
||||||
|
# spider('https://m.weibo.cn/status/4489758501815678?')
|
||||||
|
queue.join()#等待下载完成
|
||||||
Loading…
Reference in New Issue
Block a user