| 
1
2
3
4
5
6
import pandas as pd
import matplotlib.pyplot as plt

df = pd.read_csv('datasets/avocado.csv')

df.head()

1
df.tail()

1
df['AveragePrice'].head()
0    1.33
1    1.35
2    0.93
3    1.08
4    1.28
Name: AveragePrice, dtype: float64

1
df.AveragePrice.head()
0    1.33
1    1.35
2    0.93
3    1.08
4    1.28
Name: AveragePrice, dtype: float64

1
2
albany_df = df[df['region'] == 'Albany']
albany_df.head()

1
albany_df.index
Int64Index([    0,     1,     2,     3,     4,     5,     6,     7,     8,
                9,
            ...
            17603, 17604, 17605, 17606, 17607, 17608, 17609, 17610, 17611,
            17612],
           dtype='int64', length=338)

1
2
albany_df = albany_df.set_index('Date')
albany_df.head()

1
albany_df.index
Index(['2015-12-27', '2015-12-20', '2015-12-13', '2015-12-06', '2015-11-29',
       '2015-11-22', '2015-11-15', '2015-11-08', '2015-11-01', '2015-10-25',
       ...
       '2018-03-11', '2018-03-04', '2018-02-25', '2018-02-18', '2018-02-11',
       '2018-02-04', '2018-01-28', '2018-01-21', '2018-01-14', '2018-01-07'],
      dtype='object', name='Date', length=338)

1
albany_df.plot()
<matplotlib.axes._subplots.AxesSubplot at 0x15c18c71e88>


1
albany_df['AveragePrice'].plot()
<matplotlib.axes._subplots.AxesSubplot at 0x15c18ad00c8>

|

本文加载摄像头，显示视频，并写入文件 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
import numpy as np
import cv2

cap = cv2.VideoCapture(0)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('output.avi',fourcc, 20.0, (640,480))

while(True):
    ret, frame = cap.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    out.write(frame)
    cv2.imshow('frame',gray)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
out.release()
cv2.destroyAllWindows()

|

上文已经安装了opencv的开发环境

这里测试图像显示及写入 

1
2
3
4
5
6
7
8
9
10
11
# 导入cv模块
import cv2 as cv
# 读取图像，支持 bmp、jpg、png、tiff 等常用格式
img = cv.imread("data/lena.jpg", cv.IMREAD_COLOR)
# 创建窗口并显示图像
cv.imshow("Image", img)
cv.waitKey(0)
# 释放窗口
cv.destroyAllWindows()

cv.imwrite('lena2.jpg', img)

|

scikit-learn是一个python版的机器学习库，它依赖于numpy scipy matplotlib等

其中numpy封装了矩阵相关数据结构及其操作，scipy封装了最优化、数值积分、线性代数等相关模块，matplotlib貌似是一个绘图工具

而scikit-learn作为机器学习库，依赖于上面这些功能模块

首先需要安装环境：

conda install scikit-learn

上面的命令会自动安装相关依赖（numpy scipy）

conda install matplotlib

|

本文实现用户登录功能 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
import scrapy
from scrapy.http import FormRequest
from scrapy.utils.response import open_in_browser


class BlogSpider(scrapy.Spider):
    name = 'quotes'
    start_urls = {
        'http://quotes.toscrape.com/login'
    }

    def parse(self, response):
        token = response.css('form input::attr(value)').extract_first()
        return FormRequest.from_response(response, formdata={
            'csrf_token': token,
            'username': 'dsadsa',
            'password': 'dsads'
        }, callback=self.start_scraping)

    def start_scraping(self, response):
        # open_in_browser(response)
        all_div_quotes = response.css('div.quote')
        for quote in all_div_quotes:
            title = quote.css('span.text::text').extract()
            author = quote.css('.author::text').extract()
            tag = quote.css('.tag::text').extract()
            yield {'标题': title, '作者': author, '标签': tag}

补充：
调试scrapy爬虫
scrapytutorial/scrapytutorial目录，新建run.py文件

1
2
3
4
5
6
from scrapy import cmdline


name = 'quotes'
cmd = 'scrapy crawl {0}'.format(name)
cmdline.execute(cmd.split())

|

设置爬虫的user-agent

安装scrapy-user-agents
conda install scrapy-user-agents

setting.py文件添加

1
2
3
4
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy_user_agents.middlewares.RandomUserAgentMiddleware': 400,
}

设置爬虫代理

安装scrapy-proxy-pool
conda install scrapy-proxy-pool

setting.py文件添加

1
2
3
4
5
6
7
8
PROXY_POOL_ENABLED = True

DOWNLOADER_MIDDLEWARES = {
    # ...
    'scrapy_proxy_pool.middlewares.ProxyPoolMiddleware': 610,
    'scrapy_proxy_pool.middlewares.BanDetectionMiddleware': 620,
    # ...
}

|

本文实现详情页数据的采集功能

首先修改items.py文件，新增ArticleItem类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import scrapy

class ScrapytutorialItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    link = scrapy.Field()
    # pass
	

class ArticleItem(scrapy.Item):
    title = scrapy.Field()
    link = scrapy.Field()
    content = scrapy.Field()
    update = scrapy.Field()
    cate = scrapy.Field()

修改blog_spider.py文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import scrapy
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

from ..items import ScrapytutorialItem
from ..items import ArticleItem


class BlogSpider(scrapy.Spider):
    name = 'blog'    
    start_urls = {        
        'https://www.cnblogs.com/chenying99'
    }
    def parse(self, response):

        items = ScrapytutorialItem()
        container = response.css('#main')[0]
        posts = container.css('div.post')
        for article in posts:

            title = article.css('a.postTitle2::text').extract_first().strip()
            link = article.css('a.postTitle2::attr(href)').extract_first()
            items['title'] = title
            items['link'] = link
            url = response.urljoin(link)
            yield scrapy.Request(url=url, callback=self.parse_details)           

        
        next_page = response.css('#homepage_bottom_pager > div > a:nth-child(8)::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, callback=self.parse)

    def parse_details(self, response):
        items = ArticleItem()        
        title = response.css('#cb_post_title_url::text').extract_first(default='not-found').strip()
        link = response.css('#cb_post_title_url::attr(href)').extract_first().strip()
        content = response.css('#cnblogs_post_body').extract_first().strip()
        cate = response.css('#BlogPostCategory > a::text').get(default='not-found')       

        update = response.css('#post-date::text').extract_first().strip()
        items['title'] = title
        items['link'] = link
        items['content'] = content
        items['cate'] = cate
        items['update'] = update
        yield items

修改pipelines.py文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
import pymongo
from .items import ArticleItem


class ScrapytutorialPipeline(object):

    def __init__(self):
        self.conn = pymongo.MongoClient(
            '127.0.0.1',
            27017
        )
        db = self.conn['scrapy']
        self.collection = db['article']

    def process_item(self, item, spider):        
        if isinstance(item, ArticleItem):
            return self.collection.insert(dict(item))
        return item

|

本文实现scrapy的分页采集功能

分页方式一：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import scrapy
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

from ..items import ScrapytutorialItem


class BlogSpider(scrapy.Spider):
    name = 'blog'    
    start_urls = {
        'https://www.cnblogs.com/chenying99/'        
    }
    def parse(self, response):

        items = ScrapytutorialItem()
        container = response.css('#main')[0]
        posts = container.css('div.post')
        for article in posts:

            title = article.css('a.postTitle2::text').extract_first().strip()
            link = article.css('a.postTitle2::attr(href)').extract_first()
            items['title'] = title
            items['link'] = link            
            yield items
        
        next_page = response.css('#homepage_bottom_pager > div > a:nth-child(8)::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, callback=self.parse)

分页方式二：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
import scrapy
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

from ..items import ScrapytutorialItem


class BlogSpider(scrapy.Spider):
    name = 'blog'
    page_number = 2
    start_urls = {        
        'https://www.cnblogs.com/chenying99/default.html?page=1'
    }
    def parse(self, response):

        items = ScrapytutorialItem()
        container = response.css('#main')[0]
        posts = container.css('div.post')
        for article in posts:

            title = article.css('a.postTitle2::text').extract_first().strip()
            link = article.css('a.postTitle2::attr(href)').extract_first()
            items['title'] = title
            items['link'] = link                   
            yield items

        next_page = 'https://www.cnblogs.com/chenying99/default.html?page='+str(BlogSpider.page_number)
        if BlogSpider.page_number < 10:
            BlogSpider.page_number += 1
            yield response.follow(next_page, callback=self.parse)

|

也可以将scrapy采集的数据输出到json文件，运行scrapy crawl blog -o items.json

本文主要介绍怎样将采集的数据存储到mongodb

首先配置setting.py文件里面的item pipelines 

1
2
3
4
5
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'scrapytutorial.pipelines.ScrapytutorialPipeline': 300,
}

安装pymongo
conda install pymongo

修改pipelines.py文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import pymongo

class ScrapytutorialPipeline(object):

    def __init__(self):
        self.conn = pymongo.MongoClient(
            '127.0.0.1',
            27017
        )
        db = self.conn['scrapy']
        self.collection = db['blog']

    def process_item(self, item, spider):
        # print("pipline测试 "+item['title'])
        self.collection.insert(dict(item))

运行scrapy crawl blog
查看mongodb数据库记录