時(shí)間：2023-05-20 10:36:02 | 來源：網(wǎng)站運(yùn)營(yíng)
時(shí)間：2023-05-20 10:36:02 來源：網(wǎng)站運(yùn)營(yíng)
python 爬蟲 爬微博 分析 數(shù)據(jù)：
在線咨詢
import jsonimport scrapyfrom weibo.items import WeiboItemfrom bs4 import BeautifulSoupclass weibo_spider(scrapy.Spider): name = "weibo" start_urls =["https://m.weibo.cn/api/container/getIndex?uid=1927305954&t=0&luicode=10000011&lfid=100103type%3D1%26q%3D%E6%88%90%E6%9E%9C&type=uid&value=1927305954&containerid=1076031927305954"] url = "https://m.weibo.cn/api/container/getIndex?uid=1927305954&t=0&luicode=10000011&lfid=100103type%3D1%26q%3D%E6%88%90%E6%9E%9C&type=uid&value=1927305954&containerid=1076031927305954&since_id=" #start_urls = ["https://m.weibo.cn/"] allowed_domains = ["weibo.com", "weibo.cn"] since_id = "" # 下下面的id created_at = "" # 創(chuàng)建的日期 text = "" # 發(fā)布的內(nèi)容 source = "" # 發(fā)布文章的設(shè)備 scheme = "" # 原文連接 reposts_count = 0 # 轉(zhuǎn)發(fā)數(shù)量 textLength = 0 # 文章字?jǐn)?shù) comments_count = 0 # 評(píng)論個(gè)數(shù) attitudes_count = 0 # 點(diǎn)贊個(gè)數(shù) def parse(self, response): text_json = json.loads(response.body_as_unicode()) self.since_id = text_json.get('data').get('cardlistInfo').get('since_id') cards = text_json.get('data').get('cards') for it in cards: it_son = it.get('mblog') if it_son: self.created_at = it_son['created_at'] self.text = it_son['text'] self.source = it_son['source'] self.scheme = it['scheme'] self.reposts_count = it_son['reposts_count'] self.comments_count = it_son['comments_count'] self.attitudes_count = it_son['attitudes_count'] soup = BeautifulSoup(str(self.text), "html.parser") # 抓取的數(shù)據(jù)是有html標(biāo)簽 去除一下 self.text = soup.get_text() if len(self.created_at) < 6 : self.created_at = "%s%s"%("2020-", self.created_at) #由于今年的微博沒有年份 所有給數(shù)據(jù)處理一下 self.textLength = len(self.text) items = WeiboItem(created_at=self.created_at, text=self.text, source=self.source, scheme=self.scheme, reposts_count=self.reposts_count, comments_count=self.comments_count, attitudes_count=self.attitudes_count, textLength=self.textLength) # 將數(shù)據(jù)寫入items 文件中 yield items if not self.since_id: return urls = "%s%s"%(self.url, str(self.since_id)) # 獲取的下一個(gè)json鏈接 yield scrapy.Request(urls, callback=self.parse)scrapy 的 itmes.py 文件# -*- coding: utf-8 -*-# Define here the models for your scraped items## See documentation in:# https://docs.scrapy.org/en/latest/topics/items.htmlimport scrapyclass WeiboItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() since_id = scrapy.Field() # 下下面的id created_at = scrapy.Field() # 創(chuàng)建的日期 text = scrapy.Field() # 發(fā)布的內(nèi)容 source = scrapy.Field() # 發(fā)布文章的設(shè)備 scheme = scrapy.Field() # 原文連接 reposts_count = scrapy.Field() # 轉(zhuǎn)發(fā)數(shù)量 textLength = scrapy.Field() # 文章字?jǐn)?shù) comments_count = scrapy.Field() # 評(píng)論個(gè)數(shù) attitudes_count = scrapy.Field() # 點(diǎn)贊個(gè)數(shù)接下來就是導(dǎo)入數(shù)據(jù)庫了# -*- coding: utf-8 -*-# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.htmlimport pymysqlimport jsonclass WeiboPipeline(object): account = { 'user': 'root', 'password': '*******', 'host': 'localhost', 'database': 'python' } def mysqlConnect(self): connect = pymysql.connect(**self.account) return connect def __init__(self): self.connect = self.mysqlConnect() # 連接數(shù)據(jù)庫 self. cursor = self.connect.cursor(cursor = pymysql.cursors.DictCursor) #### 以json寫入 #self.fp = open("xiaofuren.json", 'w', encoding='utf-8') def insertMsg(self, scheme, text, source, reposts_count, comments_count, attitudes_count, textLength, created_at): try: self.cursor.execute( "INSERT INTO %s VALUES( /'%s/' ,/' %s/' ,/' %s/',/' %d/',/' %d/',/' %d/',/' %d/',/' %s/')" % ( "weibo", scheme, text, source, reposts_count, comments_count, attitudes_count, textLength, created_at) ) self.connect.commit() except Exception as e: print("insert_sql error: " + e) def open_spider(self, spider): print("爬蟲開始了******************") def process_item(self, item, spider): self.insertMsg( item['scheme'], item['text'], item['source'], item['reposts_count'], item['comments_count'], item['attitudes_count'], item['textLength'], item['created_at']) return item #### 以json寫入 # itme_json = json.dumps(dict(item), ensure_ascii=False) # self.fp.write(itme_json + '/n') # return item def close_spider(self, spider): print("爬蟲結(jié)束***************") print("數(shù)據(jù)寫入成功") self.cursor.close() # since_id = "" # 下下面的id # created_at = "" # 創(chuàng)建的日期 # text = "" # 發(fā)布的內(nèi)容 # source = "" # 發(fā)布文章的設(shè)備 # scheme = "" # 原文連接 # reposts_count = 0 # 轉(zhuǎn)發(fā)數(shù)量 # textLength = 0 # 文章字?jǐn)?shù) # comments_count = 0 # 評(píng)論個(gè)數(shù) # attitudes_count = 0 # 點(diǎn)贊個(gè)數(shù)import datetimeimport pymysqlaccount = { 'user' : 'root', 'password' : 'zhaobo123..', 'host' : 'localhost', 'database' : 'python'}def mysqlConnect(account): connect = pymysql.connect(**account) return connectdef getMessage(cursor, month, day, year, phone, dianzan, zhuanfa, pinlun, textLength, dates): sql = 'select * from weibo ORDER BY created_at' cursor.execute(sql) row = cursor.fetchall() Day = {} #建立字典便于統(tǒng)計(jì)每天發(fā)送的微博 Year = {} Month = {} for i in range(1, 32): Day[i] = 0 for i in range(1, 13): Month[i] = 0 for i in range(2013, 2021): Year[i] = 0 for it in row: date = datetime.datetime.strptime(it['created_at'], " %Y-%m-%d") Year[date.year] += 1 Day[date.day] += 1 Month[date.month] += 1 phone.append(it['source']) dianzan.append(it['attitudes_count']) zhuanfa.append(it['reposts_count']) pinlun.append(it['comments_count']) textLength.append(it['textLength']) dates.append(it['created_at']) for i in range(1, 32): day.append(Day[i]) for i in range(1, 13): month.append(Month[i]) for i in range(2013, 2021): year.append(Year[i])if __name__ == '__main__': month = [] # 按照月發(fā)送的微博 year = [] # 按照年發(fā)送的微博 day = [] # 按照日發(fā)送的微博 phone = [] # 手機(jī)的種類 dianzan = [] # 點(diǎn)贊數(shù) zhuanfa = [] # 轉(zhuǎn)發(fā)數(shù) pinlun = [] # 評(píng)論數(shù) textLength = [] #發(fā)送微博長(zhǎng)度 dates = [] # 時(shí)間 connect = mysqlConnect(account) cursor = connect.cursor(cursor=pymysql.cursors.DictCursor) getMessage(cursor, month, day, year, phone, dianzan, zhuanfa, pinlun, textLength, dates)代碼里有注釋我就不解釋了。#按照日 發(fā)微博的個(gè)數(shù) xday = [] for i in range(1, 32): xday.append(i) bar = ( Bar() .add_xaxis(xday) .add_yaxis("每天發(fā)送的微博", day) .set_global_opts(title_opts=opts.TitleOpts(title="狗哥發(fā)微博統(tǒng)計(jì)")) ) bar.render(path= 'day.html') # 按月 xmonth = [] for i in range(1, 13): xmonth.append(i) bar = ( Bar() .add_xaxis(xmonth) .add_yaxis("每月發(fā)送的微博", month) .set_global_opts(title_opts=opts.TitleOpts(title="狗哥發(fā)微博統(tǒng)計(jì)")) ) bar.render(path = 'month.html') # 按年 xyear = [] for i in range(2013, 2021): xyear.append(i) bar = ( Bar() .add_xaxis(xyear) .add_yaxis("每年發(fā)送的微博", year) .set_global_opts(title_opts=opts.TitleOpts(title="狗哥發(fā)微博統(tǒng)計(jì)")) ) bar.render(path = 'year.html')關(guān)鍵詞：分析,數(shù)據(jù),爬蟲
客戶&案例
營(yíng)銷資訊
關(guān)于我們
客戶&案例
營(yíng)銷資訊
關(guān)于我們
微信公眾號(hào)
版權(quán)所有? 億企邦 1997-2025 保留一切法律許可權(quán)利。
