forked from Boris-code/feapder
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmongo_spider.py
More file actions
39 lines (32 loc) · 1011 Bytes
/
Copy pathmongo_spider.py
File metadata and controls
39 lines (32 loc) · 1011 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# -*- coding: utf-8 -*-
"""
Created on 2021-02-08 16:06:12
---------
@summary:
---------
@author: Boris
"""
import feapder
from feapder import Item, UpdateItem
class TestMongo(feapder.AirSpider):
__custom_setting__ = dict(
ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"],
MONGO_IP="localhost",
MONGO_PORT=27017,
MONGO_DB="feapder",
MONGO_USER_NAME="",
MONGO_USER_PASS="",
)
def start_requests(self):
yield feapder.Request("https://www.baidu.com")
def parse(self, request, response):
title = response.xpath("//title/text()").extract_first() # 取标题
for i in range(10):
item = Item() # 声明一个item
item.table_name = "test_mongo"
item.title = title + str(666) # 给item属性赋值
item.i = i + 5
item.c = "777"
yield item # 返回item， item会自动批量入库
if __name__ == "__main__":
TestMongo().start()