aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCody Hiar <codyfh@gmail.com>2018-03-20 17:56:41 -0600
committerCody Hiar <codyfh@gmail.com>2018-03-20 17:56:41 -0600
commitf8411cd5804d00ea4c4b38478ede487a847425bd (patch)
tree4490c7433bee5b1fe3a64243dafafbdfca8988d5
parent26240275ee4ef5fcd9fc1727edcd30cce4f004ab (diff)
Adding in pipeline
-rw-r--r--myproject/myproject/pipelines.py15
-rw-r--r--myproject/myproject/settings.py6
-rw-r--r--myproject/myproject/spiders/blogitems.py3
3 files changed, 14 insertions, 10 deletions
diff --git a/myproject/myproject/pipelines.py b/myproject/myproject/pipelines.py
index 0bdee9f..4cf9b9e 100644
--- a/myproject/myproject/pipelines.py
+++ b/myproject/myproject/pipelines.py
@@ -1,11 +1,16 @@
# -*- coding: utf-8 -*-
+import json
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+class FileWritePipeline(object):
+
+ def open_spider(self, spider):
+ self.file = open('items.json', 'w')
+
+ def close_spider(self, spider):
+ self.file.close()
-class MyprojectPipeline(object):
def process_item(self, item, spider):
+ line = json.dumps(dict(item)) + "\n"
+ self.file.write(line)
return item
diff --git a/myproject/myproject/settings.py b/myproject/myproject/settings.py
index 6790f78..cbd1ed9 100644
--- a/myproject/myproject/settings.py
+++ b/myproject/myproject/settings.py
@@ -65,9 +65,9 @@ LOG_LEVEL = 'WARNING'
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-# 'myproject.pipelines.MyprojectPipeline': 300,
-#}
+ITEM_PIPELINES = {
+ 'myproject.pipelines.FileWritePipeline': 300,
+}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
diff --git a/myproject/myproject/spiders/blogitems.py b/myproject/myproject/spiders/blogitems.py
index 805681b..973f022 100644
--- a/myproject/myproject/spiders/blogitems.py
+++ b/myproject/myproject/spiders/blogitems.py
@@ -28,5 +28,4 @@ class BlogItemsSpider(scrapy.Spider):
date=date,
year=year
)
- print(blog_post)
-
+ yield blog_post