[关闭]
@Scrazy 2017-04-01T13:44:15.000000Z 字数 5191 阅读 890

新浪博客文本聚类

python

前言

这是年前帮一位 QQ 好友完成论文报告所写的程序。有待完善。

以下是本次报告所使用的程序, 全部使用 Python 编写。根据需要，编写了以下四个程序。

  1. article_base_info.py 用于抓取文章的基本信息：文章标题、链接、作者、发表日期
  2. article_content_gevent.py 用于抓取文章内容
  3. text_category.py 对文章进行分类
  4. format_data.py 格式化数据

下面是程序代码

  1. # coding: utf-8
  2. '''
  3. 程序： article_base_info.py
  4. 1. 此程序通过给定的页数抓取新浪博客文章的基本信息：文章标题、链接、作者、发表日期
  5. 2. 数据保存到MongoDB中
  6. '''
  7. import re
  8. import concurrent.futures
  9. import requests
  10. from bs4 import BeautifulSoup as bs
  11. from pymongo import MongoClient
  14. def fetch(url):
  15.     res = requests.get(url)
  16.     res.encoding = 'gbk'
  17.     content = bs(res.text, 'lxml')
  18.     return content
  21. def base_info(html):
  22.     pattern = re.compile(r'http://blog.sina.com.cn/s/blog_.*\.html')
  23.     links = re.findall(pattern, str(html))
  24.     date_ = re.findall(r'\((\d{2,}.*)\)', str(html))
  25.     tle_auth = html.select('li')
  26.     authes = (auth.text.split(' ')[0] for auth in tle_auth)
  27.     titles = (title.text.split(' ')[-1] for title in tle_auth)
  28.     for infos in zip(links, titles, authes, date_):
  29.         yield infos
  32. def save(url):
  33.     html = fetch(url)
  34.     data = base_info(html)
  35.     client = MongoClient('localhost', 27017)
  36.     db = client.infos
  37.     coll = db.coll
  38.     for num, d in enumerate(data, 1):
  39.         datum = {
  40.             'links': d[0],
  41.             'title': d[1],
  42.             'auther': d[2],
  43.             'date': d[3]
  44.         }
  46.         count = coll.find({'links': d[0]}).count()
  47.         if count == 0:
  48.             coll.insert_one(datum)
  49.     print('{} is grabbed'.format(urls))
  52. if __name__ == '__main__':
  53.     url = 'http://roll.blog.sina.com.cn/list/other/index_{}.shtml'
  55.     start = int(input('请输入开始页数, 默认为1 >> '))
  56.     if not start:
  57.         start = 1
  59.     end = int(input('输入结束页数， 默认为100 >> '))
  60.     if not end:
  61.         end = 100
  63.     pages = range(start, end + 1)
  64.     urls = [url.format(page) for page in pages]
  66.     with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
  67.         executor.map(save, urls)
  1. # -*-coding: utf-8 -*-
  2. '''
  3. 程序： article_content_gevent.py
  4. 1. 此程序是用来抓取新浪博客的文章内容的!
  5. 2.文章链接从 *筛选后所有博客数据.csv* 读取, 此 .csv 文件由 article_base_info.py 抓取生成。
  6. 3. 由于作者删帖或者其他方面的原因,导致文章本身已不存在而其链接仍存在的现象。
  7. '''
  9. import os
  10. import csv
  11. import logging
  12. import requests
  13. import gevent
  14. from bs4 import BeautifulSoup as bs
  17. def fetch(url):
  18.     res = requests.get(url)
  19.     res.encoding = 'utf-8'
  20.     content = bs(res.text, 'lxml')
  21.     if not content:
  22.         logging.warning('The blog have been deleted!')
  23.     return content
  26. def content_get(html):
  27.     try:
  28.         artical = html.select('#sina_keyword_ad_area2')[0].text.strip()
  29.     except IndexError as e:
  30.         print(e)
  31.         logging.warning('the page is None')
  32.         artical = ' '
  33.     return artical
  36. def links_get(filename, urls=None):
  37.     with open(filename, 'r') as csvfile:
  38.         logging.info('readed the file {}'.format(filename))
  39.         reader = csv.reader(csvfile)
  40.         if urls is None:
  41.             urls = []
  42.         urls = [row[0] for row in reader]
  43.     return urls
  46. def download(url):
  47.     html = fetch(url)
  48.     artical = content_get(html)
  49.     with open('/home/mouse/Documents/artical/{}.txt'
  50.               .format(url[-12:-5]), 'w') as f:
  51.         f.write(artical)
  52.     logging.info('writring the {}'.format(url))
  55. if __name__ == '__main__':
  56.     logging.basicConfig(format='%(asctime)s %(message)s',
  57.                         level=logging.WARNING)
  58.     filename = '/home/mouse/我的坚果云/董姐的论文所需/筛选后所有博客数据.csv'
  59.     urls = links_get(filename)
  60.     if not os.path.isdir('/home/mouse/Documents/artical/'):
  61.         os.makedirs('/home/mouse/Documents/artical/')
  62.     threads = [gevent.spawn(download, url) for url in urls]
  63.     gevent.joinall(threads)
  1. # coding=utf-8
  2. '''
  3. 程序： text_category.py
  4. 1. 此程序用于对从新浪博客抓取的文章进行自动分类
  5. 2. 分类所使用的库来自 https://github.com/2shou/TextGrocery
  6. 3. 工作流 读取已抓取的文章标题 -> 对文章分类 -> 写入分类后的文章标题
  7. '''
  8. import os
  9. import csv
  10. from tgrocery import Grocery
  11. from train_txt import train_src
  14. def category(title_lst, cates=None):  # 对文章分类
  15.     if cates is None:
  16.         cates = []
  17.     for title in title_lst:
  18.         cate = new_grocery.predict(title)
  19.         cates.append(cate.predicted_y)
  20.     return cates
  23. def get_artical_title(filename, title_lst=None):  # 读取文本
  24.     if title_lst is None:
  25.         title_lst = []
  27.     with open(filename, 'r') as f1:
  28.         f1_csv = csv.reader(f1)
  29.         title_lst = [row[1] for row in f1_csv]
  31.     return title_lst
  34. def write_cated_info(filename, new_filename):  # 写入已分类的文章
  35.     titles = get_artical_title(filename)
  36.     categ = category(titles)
  37.     with open(filename, 'r') as read_file:
  38.         reader = csv.reader(read_file)
  39.         for i, row in enumerate(reader):
  40.             row.append(categ[i])
  41.             with open(new_filename, 'a+') as write_file:
  42.                 writer = csv.writer(write_file)
  43.                 writer.writerow(row)
  45.             print 'writing the {} item'.format(i)
  46.     print 'Done....................'
  49. if __name__ == "__main__":
  50.     # filename和new_filename是文件路径，保存读取和写入的文件
  51.     # 更改路径名即可对不同的数据分类，前提要符合一定格式
  52.     filename = '/home/mouse/我的坚果云/董姐的论文所需/female7.csv'
  53.     new_filename = '/home/mouse/我的坚果云/董姐的论文所需/female7_2.csv'
  54.     if os.path.isfile(new_filename):
  55.         os.remove(new_filename)
  56.     grocery = Grocery('sample')
  57.     grocery.train(train_src)
  58.     grocery.save()
  59.     new_grocery = Grocery('sample')
  60.     new_grocery.load()
  61.     write_cated_info(filename, new_filename)
  1. # -*- coding: utf-8 -*-
  2. '''
  3. 程序： format_data.py
  4. 此程序是一个辅助程序，用于对 *标题整理数据.xlsx* 的格式化，标题整理数据转换为 csv 格式
  5. '''
  7. import csv
  8. from collections import namedtuple
  9. cate = ['社会冲突和问题', '毛泽东思想与政策', '政党与政府设置', '民主与法治', '民族和国际关系',
  10.         '媒体与言论自由', '资本主义与市场经济', '全球化和对外开放', '民生与福利',
  11.         '家庭冲突与伦理', '传统文化', '性与个人自由', '环境污染', '生态保护', ]
  12. Category = namedtuple(
  13.     'Category', 'social mao govm demcy nation media capi glob live home tran sex env eco')
  15. filename = '/home/mouse/我的坚果云/董姐的论文所需/标题整理数据2.csv'
  18. def train_text(filename, train_src=None):
  19.     if train_src is None:
  20.         train_src = []
  22.     def format_cate():
  23.         for emp in map(Category._make, csv.reader(open(filename, 'r'))):
  24.             social = (cate[0], emp.social)
  25.             mao = (cate[1], emp.mao)
  26.             govm = (cate[2], emp.govm)
  27.             demcy = (cate[3], emp.demcy)
  28.             nation = (cate[4], emp.nation)
  29.             media = (cate[5], emp.media)
  30.             capi = (cate[6], emp.capi)
  31.             glob = (cate[7], emp.glob)
  32.             live = (cate[8], emp.live)
  33.             home = (cate[9], emp.home)
  34.             tran = (cate[10], emp.tran)
  35.             sex = (cate[11], emp.sex)
  36.             env = (cate[12], emp.env)
  37.             eco = (cate[13], emp.eco)
  38.             yield social, mao, govm, demcy, nation, media, capi, glob, \
  39.                 live, home, tran, sex, env, eco
  41.     for cat in format_cate():
  42.         train_src.extend(list(cat))
  44.     return train_src

以上程序均有本人编写，并且全部我的电脑上运行通过，但未在其他电脑和平台上测试，由于各种依赖和兼容性问题以及本人水平有限，不保证他人也能正常运行此程序。

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注