Python爬蟲(chóng)實(shí)例分析

import urllib.parse

import urllib.request

# params  CategoryId=808 CategoryType=SiteHome ItemListActionName=PostList PageIndex=3 ParentCategoryId=0 TotalPostCount=4000

def getHtml(url,values):

    user_agent='Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36'

    headers = {'User-Agent':user_agent}

    data = urllib.parse.urlencode(values)

    response_result = urllib.request.urlopen(url+'?'+data).read()

    html = response_result.decode('utf-8')

    return html

#獲取數(shù)據(jù)

def requestCnblogs(index):

    print('請(qǐng)求數(shù)據(jù)')

    url = 'http://www.cnblogs.com/mvc/AggSite/PostList.aspx'

    value= {

         'CategoryId':808,

         'CategoryType' : 'SiteHome',

         'ItemListActionName' :'PostList',

         'PageIndex' : index,

         'ParentCategoryId' : 0,

        'TotalPostCount' : 4000

    }

    result = getHtml(url,value)

    return result

<div class="post_item">

    <div class="digg">

        <div class="diggit" onclick="DiggPost('hyper-xl',6417741,281238,1)">

            <span class="diggnum" id="digg_count_6417741">1</span>

        </div>

        <div class="clear"></div>

        <div id="digg_tip_6417741" class="digg_tip"></div>

    </div>

    <div class="post_item_body">

        <h4><a class="titlelnk" href="http://www.cnblogs.com/hyper-xl/p/6417741.html" target="_blank">Python 字符串格式化</a></h4>

        <p class="post_item_summary">

            <a href="http://www.cnblogs.com/hyper-xl/" target="_blank">

                <img width="48" height="48" class="pfs"

                     src="//pic.cnblogs.com/face/795666/20160421231717.png" alt="" />

            </a>    轉(zhuǎn)載請(qǐng)注明出處 Python2.6+ 增加了str.format函數(shù)，用來(lái)代替原有的'%'操作符

            。它使用比'%'更加直觀、靈活。下面詳細(xì)介紹一下它的使用方法。 下面是使用'%'的例子: 格式很像C語(yǔ)言的printf是不是？由于'%'是一個(gè)操作符，只能在左右

            兩邊各放一個(gè)參數(shù)，因此右邊多個(gè)值需要用元組或 ...

        </p>

        <div class="post_item_foot">

            <a href="http://www.cnblogs.com/hyper-xl/" class="lightblue">新月的力量_141</a>

            發(fā)布于 2017-02-19 23:07

            <span class="article_comment">

                <a href="http://www.cnblogs.com/hyper-xl/p/6417741.html#commentform" title="" class="gray">

                    評(píng)論(0)

                </a>

            </span>

            <span class="article_view">

                <a href="http://www.cnblogs.com/hyper-xl/p/6417741.html" class="gray">

                    閱讀

                    (138)

                </a>

            </span>

        </div>

    </div>

    <div class="clear"></div>

</div>

from bs4 import BeautifulSoup

import request

import re

#解析最外層

def blogParser(index):

  cnblogs = request.requestCnblogs(index)

  soup = BeautifulSoup(cnblogs, 'html.parser')

  all_div = soup.find_all('div', attrs={'class': 'post_item_body'}, limit=20)

  blogs = []

  #循環(huán)div獲取詳細(xì)信息

  for item in all_div:

      blog = analyzeBlog(item)

      blogs.append(blog)

  return blogs

#解析每一條數(shù)據(jù)

def analyzeBlog(item):

    result = {}

    a_title = find_all(item,'a','titlelnk')

    if a_title is not None:

        # 博客標(biāo)題

        result["title"] = a_title[0].string

        # 博客鏈接

        result["href"] = a_title[0]['href']

    p_summary = find_all(item,'p','post_item_summary')

    if p_summary is not None:

        # 簡(jiǎn)介

        result["summary"] = p_summary[0].text

    footers = find_all(item,'div','post_item_foot')

    footer = footers[0]

    # 作者

    result["author"] = footer.a.string

    # 作者url

    result["author_url"] = footer.a['href']

    str = footer.text

    time = re.findall(r"發(fā)布于 .+? .+? ", str)

    result["create_time"] = time[0].replace('發(fā)布于 ','')

    comment_str = find_all(footer,'span','article_comment')[0].a.string

    result["comment_num"] = re.search(r'\d+', comment_str).group()

    view_str = find_all(footer,'span','article_view')[0].a.string

    result["view_num"] = re.search(r'\d+', view_str).group()

    return result

def find_all(item,attr,c):

    return item.find_all(attr,attrs={'class':c},limit=1)

import match

import os

import datetime

import json

def writeToTxt(list_name,file_path):

    try:

        #這里直接write item 即可，不要自己給序列化在寫入，會(huì)導(dǎo)致json格式不正確的問(wèn)題

        fp = open(file_path,"w+",encoding='utf-8')

        l = len(list_name)

        i = 0

        fp.write('[')

        for item in list_name:

            fp.write(item)

            if i<l-1:

                fp.write(',\n')

            i += 1

        fp.write(']')

        fp.close()

    except IOError:

        print("fail to open file")

#def getStr(item):

#   return json.dumps(item).replace('\'','\"')+',\n'

def saveBlogs():

    for i in range(1,2):

        print('request for '+str(i)+'...')

        blogs = match.blogParser(i,5)

        #保存到文件

        path = createFile()

        writeToTxt(blogs,path+'/blog_'+ str(i) +'.json')

        print('第'+ str(i) +'頁(yè)已經(jīng)完成')

    return 'success'

def createFile():

    date = datetime.datetime.now().strftime('%Y-%m-%d')

    path = '/'+date

    if os.path.exists(path):

        return path

    else:

        os.mkdir(path)

        return path

result = saveBlogs()

print(result)