From e52f8f0c6bd258898de7bfad08b341f82d2a1e1e Mon Sep 17 00:00:00 2001
From: JJ D'Aurora <jjdaurora@JJs-MacBook-Pro.attlocal.net>
Date: Mon, 9 Mar 2020 21:16:16 -0700
Subject: [PATCH 1/5] parellel dots

---
 news_api/__pycache__/items.cpython-38.pyc     | Bin 470 -> 491 bytes
 news_api/__pycache__/pipelines.cpython-38.pyc | Bin 2026 -> 2026 bytes
 news_api/items.py                             |   3 ++-
 news_api/pipelines.py                         |   2 +-
 news_api/settings.cfg                         |   4 ++++
 .../__pycache__/news_agg.cpython-38.pyc       | Bin 1972 -> 2148 bytes
 news_api/spiders/news_agg.py                  |  18 ++++++++++++++++++
 7 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 news_api/settings.cfg

diff --git a/news_api/__pycache__/items.cpython-38.pyc b/news_api/__pycache__/items.cpython-38.pyc
index 1f2c692f849f675755735f2ec63b897a9d4079bb..d68d6cc39456fffd2759f114aca147de8bbd03fa 100644
GIT binary patch
delta 120
zcmcb{{F<3Bl$V!_0SGjO)8qJ<Ch{dP`b^AsmP=(`z_O4biYJ9Jm_d{27Pnt&d9h<b
zre{fNuAe6F#1pw9w>XPa^GY&vfy7FNA_1TgMFNxU7+pAofJ_b+4#vqHjA{Z9aj+za
K5S+Y^Q33!qf*gwg

delta 119
zcmaFOe2tkel$V!_0SLbQiHdvAIFT=b(Pd)3vsemaFoP!3EpETm@?yt=OwW?kTt7{o
zi5GG?S27gw1JxJtPYz^sVG{r{IansoWmMCEh=Qd+1W4u<hfQvNN@-529S4wC%mgGj
Fm;rDg8(sha

diff --git a/news_api/__pycache__/pipelines.cpython-38.pyc b/news_api/__pycache__/pipelines.cpython-38.pyc
index 5ebffe35037321167f06daaac0b8e0335a810974..cd21fcc67edb8bc6c3703b90cb625c6dd3df76b6 100644
GIT binary patch
delta 22
ccmaFG|B9bCl$V!_0SML#rpMWB<h{!d07C=?+yDRo

delta 22
ccmaFG|B9bCl$V!_0SNf?V&cp;^4?_!06pLZM*si-

diff --git a/news_api/items.py b/news_api/items.py
index c5cb08a..9e8cd11 100644
--- a/news_api/items.py
+++ b/news_api/items.py
@@ -18,6 +18,7 @@ class NewsApiItem(scrapy.Item):
     description = scrapy.Field()
     url = scrapy.Field()
     publishedat = scrapy.Field()
-    content = scrapy.Field()    
+    content = scrapy.Field() 
+    sentiment = scrapy.Field()   
     
     pass
\ No newline at end of file
diff --git a/news_api/pipelines.py b/news_api/pipelines.py
index e64be7a..041b1e0 100644
--- a/news_api/pipelines.py
+++ b/news_api/pipelines.py
@@ -61,7 +61,7 @@ def process_item(self, item, spider):
         # pdb.set_trace()
 
         self.collection.insert(dict(item))
-
+        
         logging.debug("Post added to MongoDB")
         return item
 
diff --git a/news_api/settings.cfg b/news_api/settings.cfg
new file mode 100644
index 0000000..99b25bc
--- /dev/null
+++ b/news_api/settings.cfg
@@ -0,0 +1,4 @@
+[PD]
+exist = true
+api_key = D3ZcPSa5zmgWQl4SRgmQa1jhAV9Cgi1P2BUQAFXHDKI
+
diff --git a/news_api/spiders/__pycache__/news_agg.cpython-38.pyc b/news_api/spiders/__pycache__/news_agg.cpython-38.pyc
index 7cefe6302456d35d3b401343be4e7c43eab794fc..47b8ab6d30b0f6c61dd7121346c099d2f3822563 100644
GIT binary patch
delta 611
zcmX|-O>5LZ7{_Ol$z+nvuB|QN%KCyH>b3~B)`Qp{szogY>2~Q0Ik=>mVVgCZ#F_D>
zWtSf6QG_L+*Y@IF5d02a^^~(8!4Ke*Am(BI^ZQT!&&(v>F8rQq?9}Tng6H#0|HXGU
zX|!-QS=gNCh~wDt9pRk0oStKho+55@_8xI2(7-%3{Tjzz)OIExSMK-=>-YR{s~g-u
z9`^S-@p5;&KkNjHgTuAm2ao&F;?~lmozB{m{b%c&8()9432x?p$is#W2U{>g80t&D
zYF$40Fg8afH=1ZnxH-18c}7NLWStvhtgTZtrtp?D(TH+u1p(70+^h2Tf6pq<D(_V6
zR1CX9pHZ<}v0HJi;u<i%3jq&+?jnp{&2j{vSby%bK<X%rg(_-Xs8B|kj?$zsM3Vos
zmQQ5)3ky3;6D<;5I0`fxg0yKt1`3i^oCaJK<RIlog_i{~h+`4+R4dtnUP(aOwwx~a
zVN$3pO_Y$=OKZIe;@6@eTD;S~69-AZ7p7eJED>+jh89Dab2>A36SC?W2*RG>84MHb
z5o&mN3d;Le%*;R0ReUA?MYoq`%MK5u%(k!#Ip#<T3vEv}V3Jos3Nzye(kSg*zHi@}
iz5(dhl+e8(i+V4`QGRUSzH|{*8HR!rt1_7B+J6DB)Q<W9

delta 402
zcmX|+KT88K7>AR)<dVBy&sIeHN9oX^94<mf>n1n|f{G3u3hm{FmgBA_S3ziTsJNGW
z0l$NO0l&uK=;Y|+=z9^&kGwn&c@y$}*50;)n`+fVuzo(GQJZaoeLUJ6o;S{ra0Gio
zg2iZNukE2L@HGng=4J5e;*j9F?$hId!c77@f~}tQk~SWbm7Uvbv?5DeSWUDf!X6-C
z(nNw9r~mF4cZ@T`%rH014ZDV2!=7OenA{?a?sf!%PgO34d@S>6l*DSNPo4f{Q4z6<
z<aC}-vs9Mz6)eiqJWlhF6lA8dw4g~g7OEhVOgzYPhLFm7*~w+B=2@y@c~C07A14d_
z<#akNP_+vMVc+txgMD0s_<ynV5j(*Row6(J>vz`L^JD<MvJIuM=VBtQQoH(_b#;&T
Nw#!qFA@G<9`33v%RsH|~

diff --git a/news_api/spiders/news_agg.py b/news_api/spiders/news_agg.py
index bedb77f..0f88490 100644
--- a/news_api/spiders/news_agg.py
+++ b/news_api/spiders/news_agg.py
@@ -7,6 +7,14 @@
 from news_api.items import NewsApiItem
 from datetime import datetime
 
+import paralleldots
+
+# Setting your API key
+paralleldots.set_api_key("D3ZcPSa5zmgWQl4SRgmQa1jhAV9Cgi1P2BUQAFXHDKI")
+
+# Viewing your API key
+# paralleldots.get_api_key()
+
 #API Key: 2f48e626e6bb43afa1d50e6a9cce7728
 
 class NewsApiSpider(scrapy.Spider):
@@ -27,7 +35,15 @@ def parse(self, response):
         body = json.loads(response.body)
         
         for value in body['articles']:
+            
+            text= value['description']
+            lang_code="en"
+            response=paralleldots.sentiment(text,lang_code)
+            
+            # pdb.set_trace()
+
             newsItem = NewsApiItem()
+
             newsItem['publishedat'] = value['publishedAt']
             newsItem['id'] = value['source']['id']
             newsItem['name'] = value['source']['name']
@@ -35,5 +51,7 @@ def parse(self, response):
             newsItem['description'] = value['description']
             newsItem['url'] = value['url']
             newsItem['content'] = value['content']
+            newsItem['sentiment'] = response.keys()
+            
 
             yield newsItem

From 17955e8d8c69dc939867e3573bf64f2d6be07c8f Mon Sep 17 00:00:00 2001
From: JJ D'Aurora <jjdaurora@JJs-MacBook-Pro.attlocal.net>
Date: Sun, 22 Mar 2020 01:28:44 -0700
Subject: [PATCH 2/5] keyword matching

---
 .DS_Store                                     | Bin 6148 -> 8196 bytes
 news_api/.DS_Store                            | Bin 8196 -> 8196 bytes
 news_api/__pycache__/items.cpython-38.pyc     | Bin 491 -> 511 bytes
 news_api/__pycache__/pipelines.cpython-38.pyc | Bin 2026 -> 2026 bytes
 news_api/__pycache__/settings.cpython-38.pyc  | Bin 373 -> 373 bytes
 news_api/items.py                             |   1 +
 news_api/pipelines.py                         |   7 +-
 .../__pycache__/news_agg.cpython-38.pyc       | Bin 2148 -> 2709 bytes
 news_api/spiders/news_agg.py                  |  69 ++++++++++++------
 settings.cfg                                  |   4 +
 10 files changed, 55 insertions(+), 26 deletions(-)
 create mode 100644 settings.cfg

diff --git a/.DS_Store b/.DS_Store
index 3c657618c90d77d19ae8cc252ca6c5dacd4de52f..607b25d26ca39324fe911aa99bb61142c60c1f88 100644
GIT binary patch
literal 8196
zcmeHMTWl0n82-Pdz|3&z6hXGwg_Y7!VbQkGmYce5xz$S`w50_q%k0ho6Q?uF&TLDm
z)q0CQ5R8|^L}FryFIs#d(FgTWBT-+BB%%+#81=yyjlTGV|Cuw}l(syW5G7$wa?XFw
ze>vyOcfK=c&nyw4vurjKC5cGkETq1Oi#d|8i##hOBF#dU;CP}O8B`>@+wlssykkX-
zK#V|)K#V|)K#aicfB>J_A}KdH_oX#1V+3LZ?nng0`ytI)$ZRAhqzqpjT!dEumQ}0{
z6W9435XPh<nT_Oxlwm0MDQ^#mLJ?gtAj8?77Va#wk(`iHh7)8sAsQLc4F%!o9G4E=
z30W!QGDaXqU^W8c?yita7LAZU{r;YJygb{)cG|(Y@p_oJHiv(H#4kqvU*?OB?FUUw
z-yt!tYW{+SilQmn677keQ{G5VU<75q(<mJktpi4IDj(H*J<pk{8~H=FHImbpcX@7L
z*sfIygKg7dm%)D9HNBBeuk4zB*iVl%D2l46IlXplZ1dKZOuD7DZ9J16Yi?<6%B0&{
zo5sggWo1M2gWbo6M$e8tHU9Kl;j<8;7YbF&_EdRct|vA9f|KXD3a?qPIsR~$gds6n
zf#F&O231>>+_QI|meZFF^V97w2Daz=hpdv%GN)Jjj7WFEa}Sg(Vec<^PI<`nb9zm|
za0+F|2&@jr>9NmR{y<=z2}JFP@0EhC&~d=#SfR`5dfB%QS{z^xTe~>S*P%`>Ua@LT
zedDH<_MIJ9mn^N;wPo7!{$bm<^N!UwVw=IK!-ik5T+?<>9$`S;z&>GHCAB7TS?zU=
zA?vEXw0g<iE0f8Aa^5~&wu4c<UQuLUb;;GL{-H8x@@S8Hrie5{OR_GxR#A=$eOJli
zQmVCv<a$*%lwO}FEW6mCWmMf)d!OXhm7x!4&5CZSeU6d09I4u(wJG|eYX7iAO)9r(
zJCqt^)PaIGG{p1vrTU>{R?)90$Hdr=8Gf+8Xt_MRc#6)tWb&}kcbKMSDvDB{ZIsI%
zK52cnNw!p8;##_&PoRA?$UCG+qjZkW(<^j=-la=)nLed2=n8#BU(+}A1N}rl(=YTZ
z{Z4<-pY#_14U4cCN!*LISce8|z-DYg8``lGyRirR(2e~Vz+oK4AWRIw#c7l<h8OT6
zUc$?G6Yt=COyUwgz(@E5pUJ5uDy)4(wkst*=h|GvjP_^Z*j&U+?cTHZe<J4hiv=5N
zmff{{#p?R?8=G3UKRmUNGuG|~dQIn4NF&LUkUtfENZvAMYggT)rTUmz)|r`Huj~PR
zq2Q2mRd}ySr&78gk8G;zQW~>c4Q-9<*p`SanN4h&$3&L4<`ylb&u7lcxVCmp)2o=&
z(z;_OTNf~Wr8V2ZRubDw{rCL4NR#w2bMITaMn5v|eq+x4P1i9W)mVvI)FFk{SdaU#
z5gBY^#%;wmY)4nfy1h7ngXl#c4j~5v1!kVb%queUd;}QBGk6xy;dz|HdAy3(@H*bW
z+qj6y8SLw>@J>oxnu~qIJtf!i+>>k*tkw9ODNrf=QasALt3W5{r1)Yy?ZkPH{idHd
z+>FyLM_``pF)8l<yKnye|CWy%cNilOBXIj5fT~?RyE=J3l^+!|rzY;&Y0k$vizNIe
vqzps25H3O=UdLHZ{b9)UbY#m$azaYBaQe?b1l;&rH$MO4^FQR_omTt<@xvAl

delta 117
zcmZp1XfcprU|?W$DortDU=RQ@Ie-{MGjUEV6q~50$jG`eU^gQp>t-GSZ^q4eVlx>x
zHi$DXX6N7#WCkh&0s(Fy;R@2RvG6<dWPTY(kdX|G3=AwFnh9biSZs4V&m3j|V-pcs

diff --git a/news_api/.DS_Store b/news_api/.DS_Store
index 93f448ecfd98cd9812589249e4dfcb4a041ce083..712f5722b55d7734ab064bba1f83680f1b585f64 100644
GIT binary patch
delta 1069
zcmeH_OH9;I6o&6VyxNOO8KE-+Q&1iOVTgbT@(2zA0bjwG837?`hgQ8|rognIV9fY{
z=)#zgF5DQSt~D_jU8qY9B<jM*f@pB#!i5_R#>AE0nTm1k+PgTp{r_`LoA2*za5ng0
z&h49yj7OPi)z+@Sm6o27nI(ucTkh`ZwQsp++y%v@l{NKEZ5_)SH|0pwP4ny%o4tvn
zOw$dEon(4k%ylh@Aw!MnsTSEI`vj>}7?_N(iAiQnJA;0?NR%E4!<uU82gkJJq~`Sd
z<q|;{O6r|)omWSu<#L56sY1wPGtOdKMFXN_ilI?vtj}(zb%Lac;iwwXqt3bp+9*i-
z#r`QhZn81M5u0hNkc)sgFfta4@eem0xm^wl(vom2`SJs*Y4wfke^&eDF1tOiE3rVH
zkwvmZUXoYj4S7f2lU4G8d?H`SSF%RF0iejn26(X*B`8BVs;~nMXhaj*um|1f#a{Gb
z0D~C9Ff_zqU;=TR!z|`-1y^wcw{Zt|aS!+L7*Fw>q>c-O$f?^tM_8|TVJrDhGApWU
zced=dSM$Bue{r?f@~acevd8NyD5Of58Jd1Je1>V(=|R;T(G86mr~0Ru$s$o*%1TlZ
zQr*Q{6h%tna2~}nh4R3}lvi2CJ=bq1P{W-xo{i(x)l)^v;3+tBjZKtFrRh8<$8K%o
zb|z26v4b7lCdpsMel0iai{vd?;XFT*&z$HF@)H@zfd@YLp`ZvA*oJBZP{VodLNi*>
znPS?5LpY2O!Z?ECP%*-}>YVF1=W4>jc_eTVmv9-^aNQpD+?>Bq^f1f&J>B)6-9NA^
F`~_>|`I7(u

delta 1130
zcmeH_$xjqP6vpct2Iwvq3K*6qv<{1-!{$UpSp^Y;0uwHSEK$r1-9X2oS-M#yCKDs5
z@nq5mV}iv00He_;G4bMp(U=&CF-AS82aiT0i6^X{9^@bJXdSA$>U*#2z28@JO><4R
z<};*gk+G;|n1R4MxRTt-DXD@;w@Am^+O5Tm*_d4DD-BfDHnyy8+n&ZK&Gd9nX@(Yw
zs=Z-jK-CpZAL^Rc6mz&wQ&ip51~qj;6w_U|Lwa~bUCqwPWm2A$zmes{ZsrQCNa&f2
zXycQbdDgi@@)n2!yL(azn`(M+RIz`l#3f1c?y|0CdSa5lgolJfhBm7v3ai*Fip&U|
z(ey+tt=un&OzDb-BWl!1r`0w5sJ(kioiMafJ#pA@ND$Le)-y0VGQvAB5?3`J5d^l<
zZ&%R~Hq6d3Rp({I`$_Ujt(MQz61ztpl4oRvJSQ*7Yx16aCf~>p@{{}`Yh)b)L{Myn
zghCXd7y;Cv7IkPqGuqLCAUYAkDfHts!WhCB#xVgC=P-|pxP&XPuz(v_#7!*WJ|18h
zkMRVr@CF|UANhp1V1Mutbe4@7AFXxIa>}0aeFy6gTaU7Gt*=@BSe)r!q`Vg?kLJr=
zoS8ba>51dfhGQ(%rX$#$OFQKPnb}yJZL#c=nTvbnZ2fX6Wl7u@F&<n&12RkI7C34p
ztqM@)<|a9EHLaCd3b(_N8)zeC#C|xAEXqDASWDS8>QnN9!+lFWkuT&s`OU$KNJ9?C
zng<y>up54qpbX_WfNGAo4)r*SRvbec#~i{5bfXu27=VgF3`2`Up2Rd}a2^*h$5CIl
Vf;kI!QghebUWed+uK$ed^*>=#3)TPt

diff --git a/news_api/__pycache__/items.cpython-38.pyc b/news_api/__pycache__/items.cpython-38.pyc
index d68d6cc39456fffd2759f114aca147de8bbd03fa..c91e64a4a5d7213af7d0527008f23779f048b484 100644
GIT binary patch
delta 138
zcmaFO{GXXOl$V!_0SNB@&5P5S$eYL*GBMv<E|qx!%R+`I-W0}Q22G}0+<vL$#f}A;
zo+YWdewus}FH{KM;>b>|EYB}WDPGA?BnUL1NN{o>qYH-!kjcTq!8my~qlN}V94rYU
agn+~?4x8Nkl+v73I}RYPm<dR5FarQ?eIbGX

delta 118
zcmey*{F<3Jl$V!_0SGjO)8qIi@+LC+Ow9KdOJNLV&}6#B?U!0!>{yWLS(2LTr^!3<
zMg`|eh9Uu=@*;uB$&4;+LO>=5%jETp8X6E$uoQ?81QNG6Y;yBcN^?@}IDoujCLqDV
F3;>F98BYKJ

diff --git a/news_api/__pycache__/pipelines.cpython-38.pyc b/news_api/__pycache__/pipelines.cpython-38.pyc
index cd21fcc67edb8bc6c3703b90cb625c6dd3df76b6..6aaf694bd30fcd0a00f4f3d8232ffd81f281d924 100644
GIT binary patch
delta 59
zcmaFG|B9bCl$V!_0SG28EsS&C$g9K5C^gxN`6i?MW)T)oMn>()ajZRzrjuW@?qRf?
Oyn<~3BhzG0_Jsh`C=hG_

delta 59
zcmaFG|B9bCl$V!_0SML#rpMWB<kewjl%8zGe3MaOvj~ePBcsmbIMyCUv&pYn_b^&b
OUct72k$JKw`$7P`91rUN

diff --git a/news_api/__pycache__/settings.cpython-38.pyc b/news_api/__pycache__/settings.cpython-38.pyc
index 397b5d2b5535d45fd487d092a9b8d2e39675c73d..0abfc57f32fed5d8e2214df242f0ff23329d729b 100644
GIT binary patch
delta 20
acmey$^p%M_l$V!_0SLTA(l>HvF#-TIcLZMm

delta 20
acmey$^p%M_l$V!_0SH1hV>WVUF#-TJ3j}lk

diff --git a/news_api/items.py b/news_api/items.py
index 9e8cd11..6a6313d 100644
--- a/news_api/items.py
+++ b/news_api/items.py
@@ -20,5 +20,6 @@ class NewsApiItem(scrapy.Item):
     publishedat = scrapy.Field()
     content = scrapy.Field() 
     sentiment = scrapy.Field()   
+    keywords = scrapy.Field()
     
     pass
\ No newline at end of file
diff --git a/news_api/pipelines.py b/news_api/pipelines.py
index 041b1e0..a7a165a 100644
--- a/news_api/pipelines.py
+++ b/news_api/pipelines.py
@@ -19,9 +19,8 @@ def process_item(self, item, spider):
         return item
 
 class MongoDBPipeline(object):
-
     # pdb.set_trace()
-    today = 'newsAgg_' + datetime.strftime(datetime.now(), "%Y/%m/%d") 
+    today = 'newsAgg_' + datetime.strftime((datetime.now() - timedelta(days=1)), "%Y/%m/%d")
 
 
     def __init__(self, mongo_uri, mongo_db, mongo_collection):
@@ -52,12 +51,12 @@ def open_spider(self, spider):
 
 
     def close_spider(self, spider):
-    #     ## clean up when spider is closed
+        ## clean up when spider is closed
         self.client.close()
 
     def process_item(self, item, spider):
         ## how to handle each post
-
+        
         # pdb.set_trace()
 
         self.collection.insert(dict(item))
diff --git a/news_api/spiders/__pycache__/news_agg.cpython-38.pyc b/news_api/spiders/__pycache__/news_agg.cpython-38.pyc
index 47b8ab6d30b0f6c61dd7121346c099d2f3822563..5250d9fabf4c08b1aafdb81ca565ed2a2d983b78 100644
GIT binary patch
delta 1410
zcmZWpTW=dh6rR~#dvCs_O_MlD8@EjYfzlFzgg^~9wSraxRcedGN^KbLnb;ezcb%DW
zQd_qCkh~%_vOtI@=aFZ`zu*_F;Efml1!p!*RSB==JKveJ=bJM#XZ=s@rz@4G<+4Zc
zx3+Th{Fmx!WtFPacOSgdbpw|YAV6a;@VGbgp?F3q`HFxI&ItjBlYR5p3`(cJezag@
z4lu=4)a9A05qPZF>0h6EHI)7whdi0C)6X7q52{f6kuDLKJ2L_wF2QA(hbyoEi?9TB
zScVl?g{yE4)}R5`TeYABH{j+83Cgez8)z%A`Iv0I_4|b0GA930)ZU_*1#BRP&c^q5
zHoA8<U}$$!ETJJU$OjkX#RVAz1)0W71a^}t#l;}-TTzScu~4OAFucV2DoxUE+LsLw
z(P5Od0xujixlp1Xc7`sG)1>iDw=X-xxfV;JAyW~~gl(3z_|TUs4LgmVh(dIloQX!4
zi4IpmA#6tp3*0bG*kL64Sap~lM)2PD$J0N|pRe8Dl&Mf(4t4~O`5{Zf!Dft-eb(YT
zJefYXV5O?}Y-X0^@m^;2MJz=br+y=1VfwfA)W53F+d&}abf>UA@sSmAggE3no|qyt
zI->i=866uV18AMxBQNPQXKap4FzRG%p*7JKMg<@pM|`5pnRP}e8Ckji@U>6&3-`#g
zKgY#Uaa5QU&q${z6Xe!8e@Vss1gx>G?3r^eI<n7<F+E>s=0X9Ba4_nP9AvKYkIC4>
zXt7R49@uvYa;Hwf&3*T^@8!Oi`+m;-oMYLTXXd<=^GPXpa9<cOkW;i}v@Y5TTJJmD
z*})@Kg6c80_`#!tc_7$hwQmE52^EDgmzj+hrg)+<AGi!f)YI3U89W&pw;O@q>o?;_
zwmE#E-f)=8igNlxacQD2D03wW;MajFPKOVUQUP*UNmA?$6}B6DX+MFja%SyIlrF?6
z1DRRP6b6~ugQiYbyGpQ-XSIv;C+TBsuCT3NKjI=Q^_XCB%wtHE6l;3bg6zeUNCg#a
z^sv2qktwV^h%K6^Enm!Q&k<bq(nNA`GY=lJxX(o$xmeZ{_N(N;)jhcz;zt&^C?BaO
z{Dl$?d?#=cek30#-W3ZvFNekZnqEfO#34UKAhcvu42L>;=*RGkWz(WHTA`j*G)l;7
zwDju1xanBa_w7Ymoqla^6tAIzs7|H5Ink{x^#6>|;$lgIUh0kY&x_D=HiGtK1;L>f
zU8Npfrt5~|=CQ4^nA7iF*RZa^(6Ft+L<lN+d%LU`-HvdNQdrZH<KD_^cw5JPeMuPB
U=bS74uZ-zk=SHz+IHu?R2hkO4i2wiq

delta 791
zcmZWn&ui2`6rPz(lF5&)Ten44*D955Srm$hh*EzbLg_^WX`umI(w(v0noZ)&R9)Gn
zhxX`Y3F=L{7s0cF7yl5+S@7UL;hR+SV21a-@8!+=-peCzXMfbE)@n5$!SlYcv-y>c
zr&>54UtV3nh$D^@FY<(U>~m*=F<L>~<?IM?CeWUBXhp&J!;Ju!jK^B&USEuV-SI=v
zx(BA#i);99i=Ud1$mexFeT1TbpXLqlA)k4SI<ud@;VZ;{&DtAF?fz1m7hmYR#;paF
zNqv8CJJ0g{B$horD0Xdr&ggcmk}8orE$O~Ylr(dHLb6f(wAX^B>B#^PDOQ}T2QDxh
ztP7?-0%&xMhj@=n@Q4fv$4&GAJ$W;-h88DHG@{%Z+1fg$Lpro4WQ4VSh(-=%X%h_{
zZZ9L?P7`)e#qQsjRWYk#uj1Y^T0fD6LQ*CCD&bcgW`*|wa11^K@2n$?p3iUuYpFeq
zSS)p~n+R0~T&S+><+_)pB@t=y-MxNbzPoh0S*k^<OHTp9@>AV$O1h`uqMc+hS0&xf
z_@E5)SjI^r5}s)#TTm-0K*yESMu$nE@+?(CUNE8kB2M;2Nws*TBQJ^5ovm)hMZ{9^
zQmtyymyMf7qG4dhKpkKnOw9ox9FmYQOmRpZ65=|X@XujZ@yI)aYsIeDcFw|G85V`N
zuu(H~)_{>ZQLg0w^>83znkn-a81)wRL792dWICao0w>P{lvd8SB{AV*G3Q@eya+T3
al+aso-rIU6235&_U5dUx@4)1i@BRiLMXuif

diff --git a/news_api/spiders/news_agg.py b/news_api/spiders/news_agg.py
index 0f88490..eb61bc5 100644
--- a/news_api/spiders/news_agg.py
+++ b/news_api/spiders/news_agg.py
@@ -5,7 +5,7 @@
 from scrapy import Spider
 from newsapi import NewsApiClient
 from news_api.items import NewsApiItem
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import paralleldots
 
@@ -17,41 +17,66 @@
 
 #API Key: 2f48e626e6bb43afa1d50e6a9cce7728
 
+
+
 class NewsApiSpider(scrapy.Spider):
     name = "newsagg"
     headers = {'Connection': 'keep-alive','Cache-Control': 'max-age=0','DNT': '1','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36','Sec-Fetch-User': '?1','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3','Sec-Fetch-Site': 'same-origin','Sec-Fetch-Mode': 'navigate','Accept-Encoding': 'gzip, deflate, br','Accept-Language': 'en-US,en;q=0.9',}
     newsapi = NewsApiClient(api_key='2f48e626e6bb43afa1d50e6a9cce7728')
-    today = datetime.strftime(datetime.now(), "%Y/%m/%d")
+    today = datetime.strftime((datetime.now() - timedelta(days=1)), "%Y/%m/%d")
+    keywordsToSearch = ["model s","model 3","model x","model y", "solar", "battery", "gigafactory", "autonomous driving","cybertruck", "elon musk", "ganization change", "stock price","bear market","china","conavirus","covid-19"]
+
     def start_requests(self):
+        # pdb.set_trace()
         urls = [
-           'http://newsapi.org/v2/everything?q=tesla&from=' + self.today + '&to=' + self.today + '&sortBy=popularity&apiKey=2f48e626e6bb43afa1d50e6a9cce7728'
+           'http://newsapi.org/v2/everything?q=tesla&from=' + self.today + '&to=' + self.today + '&sortBy=relevancy&language=en&apiKey=2f48e626e6bb43afa1d50e6a9cce7728'
            ]
 
         for url in urls:
            yield scrapy.Request(url=url, callback=self.parse, headers=self.headers)
 
-
     def parse(self, response):
         body = json.loads(response.body)
-        
+
         for value in body['articles']:
             
-            text= value['description']
-            lang_code="en"
-            response=paralleldots.sentiment(text,lang_code)
-            
-            # pdb.set_trace()
-
-            newsItem = NewsApiItem()
-
-            newsItem['publishedat'] = value['publishedAt']
-            newsItem['id'] = value['source']['id']
-            newsItem['name'] = value['source']['name']
-            newsItem['author'] = value['author']
-            newsItem['description'] = value['description']
-            newsItem['url'] = value['url']
-            newsItem['content'] = value['content']
-            newsItem['sentiment'] = response.keys()
+            content = value['content']
+            description = value['description']
             
+            if description is None:
+                pdb.set_trace()
+            if content is None:
+                pdb.set_trace()
+
+            for keyword in self.keywordsToSearch:
+                # pdb.set_trace()
+                if (keyword in description.lower()) or (keyword in content.lower()) :
+                        # pdb.set_trace()
+                # if "model s" in content.lower() or "model 3" in content.lower() or "model x" in content.lower() or "model y" in content.lower() or "solar" in content.lower() or "battery" in content.lower() or "gigafactory" in content.lower() or "autonomous driving" in content.lower() or "cybertruck" in content.lower() or "elon musk" in content.lower() or "organization change" in content.lower() or "stock price" in content.lower() or "bear market" in content.lower() or "china" in content.lower() or "coronavirus" in content.lower() or "covid-19" in content.lower():
+                        lang_code="en"
+                        response=paralleldots.sentiment(value['description'], lang_code)
+                        splitText=value['content'].split(". ")
+                        keywords=paralleldots.batch_keywords(splitText)
+                    
+                        # pdb.set_trace()
+
+                        newsItem = NewsApiItem()
+
+                        newsItem['publishedat'] = value['publishedAt']
+                        newsItem['id'] = value['source']['id']
+                        newsItem['name'] = value['source']['name']
+                        newsItem['author'] = value['author']
+                        newsItem['description'] = value['description']
+                        newsItem['url'] = value['url']
+                        newsItem['content'] = value['content']
+                        newsItem['sentiment'] = response
+                        newsItem['keywords'] = keywords
+                        
+
+                        yield newsItem
+
+                else:
+                    # pdb.set_trace()
+                    print("no match found")
+
 
-            yield newsItem
diff --git a/settings.cfg b/settings.cfg
new file mode 100644
index 0000000..99b25bc
--- /dev/null
+++ b/settings.cfg
@@ -0,0 +1,4 @@
+[PD]
+exist = true
+api_key = D3ZcPSa5zmgWQl4SRgmQa1jhAV9Cgi1P2BUQAFXHDKI
+

From cb0f3f02036b6267882932dab1f0b92f437240c5 Mon Sep 17 00:00:00 2001
From: JJ D'Aurora <jjdaurora@JJs-MacBook-Pro.attlocal.net>
Date: Sun, 7 Jun 2020 21:46:16 -0700
Subject: [PATCH 3/5] fix timedelta imports

---
 news_api/__pycache__/items.cpython-38.pyc     | Bin 511 -> 511 bytes
 news_api/__pycache__/pipelines.cpython-38.pyc | Bin 2026 -> 2074 bytes
 news_api/pipelines.py                         |   2 +-
 .../__pycache__/news_agg.cpython-38.pyc       | Bin 2709 -> 2725 bytes
 news_api/spiders/news_agg.py                  |   1 +
 5 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/news_api/__pycache__/items.cpython-38.pyc b/news_api/__pycache__/items.cpython-38.pyc
index c91e64a4a5d7213af7d0527008f23779f048b484..d19e7a5c271e14ef8a91b095aa6d8f3554b70139 100644
GIT binary patch
delta 20
acmey*{GXXSl$V!_0SNrK?rh}#!UzC4R0VDT

delta 20
acmey*{GXXSl$V!_0SNB@&D+TRg%JQi*9L?D

diff --git a/news_api/__pycache__/pipelines.cpython-38.pyc b/news_api/__pycache__/pipelines.cpython-38.pyc
index 6aaf694bd30fcd0a00f4f3d8232ffd81f281d924..7a2ab95fbc9f401782241b27ba00948250351572 100644
GIT binary patch
delta 689
zcmZ{hOKTKC5XZZFUcECrv!k#{6wQXj7*+*|2QMCcAb4>>5IxKxvYG0vE4w?PXCQbn
zhuwo04@=(!#ejYS!Mh-S0B;WHT|b3YolCM{L;b4$UHz!8{@Z$+#!uonWH`Qj{k4<N
z<BNQEe)ZPnvCJfA4;Xmh&lvcsF?F&KbpR5zv^tu<-d+;5gnp%};aJrTBB)W-pMRVB
z5>M6A^!U5O#D=F4lk^d1&`N=j-0B>|QzJli2d#XHP%PuXG67sUA6zu=h$$ZgIB)n0
z1NTTQvn)8SJHtAu;)gRf<yi;~2xlydAc7|PCd81SkD>LLb<?`lD<*^D{=G-TeKi_R
zR3<0tpu9U6Jp04(;x4aU=)Ww@dvTiw=C`=sp~mSY!ZKlnaQteDVsp*;#M|bFvl*S3
z{Vd#@yluBO7dQ00;Re07EA$$=+RH~psr04Abw54BH_X^yZ#NcFBop(|zj2u87KxNV
zE!HlgmSv^)>}NTX;y`<pSSOsLY?P1sWjR*W^8)lLO>Yvm2><=VvzTq8m9#>NRY!V)
zhe;$u>1s-Lo4R9mf(}1#UIiDnui%s|#@=50#qJB0SN7}e@)IqzZHvwKV9iZM>iz?e
Cz=5Iw

delta 599
zcmZ{hzfS@&6vwZ<yI$`Xpn?oQK-4JF#OUClgBk}NoHQ;MLxS{-7?9Yz#L)p36PHUj
zqk{v3v;TsVs~dlUe}Hk*zPklTY|>BP_r2Hl_4P6Hm3B{E*HYw&1~>Kn$St9Iw6a}A
z%7MakM*0dPKJBTVDYe0<)HK#s^ssKDA!Uuw4_h5S`s@)y9>WcBhMo`I*c^ugQkR&L
zGr$rQ#kwiYu>ncDat23Y59hZSe!+RgBTQvlU-1l|CZ+X|XC_5?7BiV8V>YvyBUgvH
zCrT|Po^V0D;~hkVrmmGuc_yC$WC1xqQZ!hOI93N}TKuSs_K5dwS=&Upc&k0RA<$d%
zL^^Ajm*g_ko9%Ak^SQ~qUtOSz5Tu%;6CseU_#hj1E>D3-12O<jVn~CKUqFM0>UH;)
zmmx3<0lV351VP6SFT0Evz+D9_0RHVhbRJ$Ez*Q7yRjOlT+0>$%$st&?c~RUOWt11M
iMsamXx{^9b_x#FlhVg&JnInyho8^daqoO%Dqx}ISu4q;O

diff --git a/news_api/pipelines.py b/news_api/pipelines.py
index a7a165a..d30c6e7 100644
--- a/news_api/pipelines.py
+++ b/news_api/pipelines.py
@@ -12,7 +12,7 @@
 # import ipdb
 # import pdb
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 
 class NewsApiPipeline(object):
     def process_item(self, item, spider):
diff --git a/news_api/spiders/__pycache__/news_agg.cpython-38.pyc b/news_api/spiders/__pycache__/news_agg.cpython-38.pyc
index 5250d9fabf4c08b1aafdb81ca565ed2a2d983b78..d5e75a70d7acfef166c9cca4e67bafa1e64af75f 100644
GIT binary patch
delta 149
zcmbO#x>S@ml$V!_0SNRC-i=!_kyn<nW}>!}C1Vap6h{g}3TqB$E@u>HE>{#6BLk4n
zn8O{#oyxs{CxvYxBO`-5LkfEeM+-v=M=C=Wb2D=k@5XuaSQ+Iu+p=9_VpN|j!g-BR
veDZ5f8AiUz99+h1w^&Ltb5kc<bGb-}0PW;p1VSbrMiAs+6ky_D=HdhZAkH5b

delta 136
zcmZ1~I#rZ6l$V!_0SM%!%j0HD<dtPCnW*g~$(qBF%Mrzq%NfPV$dJO2!kEJq#g)pn
zfO}$}I3v%-W%F1WWjDLBU1MTYn=He5O->AGLNOnZ;9z87lwsszlwwq2;@~V|2MTHO
eP8Q%Yp6toxA|?b@!NJJE#KQ=J0!$ptT$}(qw-z@5

diff --git a/news_api/spiders/news_agg.py b/news_api/spiders/news_agg.py
index eb61bc5..44821f0 100644
--- a/news_api/spiders/news_agg.py
+++ b/news_api/spiders/news_agg.py
@@ -5,6 +5,7 @@
 from scrapy import Spider
 from newsapi import NewsApiClient
 from news_api.items import NewsApiItem
+import time
 from datetime import datetime, timedelta
 
 import paralleldots

From fbba350629b725da6ca24b30e9abda18128e147e Mon Sep 17 00:00:00 2001
From: JJ D'Aurora <jjdaurora@JJs-MacBook-Pro.attlocal.net>
Date: Tue, 9 Jun 2020 23:34:56 -0700
Subject: [PATCH 4/5] type checking

---
 news_api/__pycache__/settings.cpython-38.pyc  | Bin 373 -> 373 bytes
 news_api/settings.cfg                         |   1 +
 news_api/settings.py                          |   2 ++
 .../__pycache__/news_agg.cpython-38.pyc       | Bin 2725 -> 2687 bytes
 news_api/spiders/news_agg.py                  |  13 +++++++------
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/news_api/__pycache__/settings.cpython-38.pyc b/news_api/__pycache__/settings.cpython-38.pyc
index 0abfc57f32fed5d8e2214df242f0ff23329d729b..69ff8248cdc40a222ab11607eee88ded79c96a14 100644
GIT binary patch
delta 22
ccmey$^p%M>l$V!_0SE-n-i`C#$eY0k07b9`{{R30

delta 22
ccmey$^p%M>l$V!_0SLTA(&Icg@@6ms06wk-2mk;8

diff --git a/news_api/settings.cfg b/news_api/settings.cfg
index 99b25bc..1ee6354 100644
--- a/news_api/settings.cfg
+++ b/news_api/settings.cfg
@@ -1,4 +1,5 @@
 [PD]
 exist = true
 api_key = D3ZcPSa5zmgWQl4SRgmQa1jhAV9Cgi1P2BUQAFXHDKI
+GOOGLE_APPLICATION_CREDENTIALS="/home/user/Downloads/service-account-file.json"
 
diff --git a/news_api/settings.py b/news_api/settings.py
index e355d84..79eae25 100644
--- a/news_api/settings.py
+++ b/news_api/settings.py
@@ -70,6 +70,8 @@
 
 }
 
+
+
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
 #AUTOTHROTTLE_ENABLED = True
diff --git a/news_api/spiders/__pycache__/news_agg.cpython-38.pyc b/news_api/spiders/__pycache__/news_agg.cpython-38.pyc
index d5e75a70d7acfef166c9cca4e67bafa1e64af75f..d95c5ad1247602b26ae8f29789e0b380c44750d3 100644
GIT binary patch
delta 254
zcmZ1~`d@@Ml$V!_0SGSFJ&3!rk=LA!QGT*3n+K!V<UTg>`q)3U%r(p@Op*+>EI^tW
zNVC>3WHGxi#6GBHt6@!H$Y!3vSj1Mt0%rk57qBd3$Yv`}r~!&Y*dR42EVb+<tP9v6
z=G3q+WU6I^n7x2Kg>@lgEeBARqlN>>;w<58W~k+=nQX*f#I0AtmBQc5$jDHrH+en#
z1V--34jdwrQ#iJZX#m|_tOIgABM%b~69*#)I~SAQWJ}ITj4G4Qa2hZ&P5#R%%X5pl
XASJ1Y38<_{f3g~v9;4diP_6&~g(f~8

delta 316
zcmew_vQ(5el$V!_0SNRC-i=$bk=LA!QEsv;n+K!H<UTg>`U#A&9JS0f%qdKg47Dsk
zni)v5*0829WOEc*l`t=0S;)Z1P{RV|d%^h(S*$J$v1e-8YS?O67qBg4$Yv|f1F~5l
z><NrTA}K7j>?Q0AI3Q-$urFk)WrWzXfHQ@4A!98EP?nRch6BjtEa9Fk&tA?QP{NbK
z-^|F!P#7?IA^QXs-dikL#rb)+Sab3dQ;Kh~B;}`6-eN9DNxH>ZoLUlJQk0mSI@y(D
zhnzalgT*EwZ!+>Q@iB2Qa)2-hGI23UO%~>y#H|LDERp~b!jm1j#3vu;<Y#1@e4SI4
Wr$_}P$^;?|CvoX9YE0JW3IG7BqfDLv

diff --git a/news_api/spiders/news_agg.py b/news_api/spiders/news_agg.py
index 44821f0..ec3f503 100644
--- a/news_api/spiders/news_agg.py
+++ b/news_api/spiders/news_agg.py
@@ -21,6 +21,7 @@
 
 
 class NewsApiSpider(scrapy.Spider):
+    
     name = "newsagg"
     headers = {'Connection': 'keep-alive','Cache-Control': 'max-age=0','DNT': '1','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36','Sec-Fetch-User': '?1','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3','Sec-Fetch-Site': 'same-origin','Sec-Fetch-Mode': 'navigate','Accept-Encoding': 'gzip, deflate, br','Accept-Language': 'en-US,en;q=0.9',}
     newsapi = NewsApiClient(api_key='2f48e626e6bb43afa1d50e6a9cce7728')
@@ -44,15 +45,15 @@ def parse(self, response):
             content = value['content']
             description = value['description']
             
-            if description is None:
-                pdb.set_trace()
-            if content is None:
-                pdb.set_trace()
+            # if description is None:
+            #     pdb.set_trace()
+            # if content is None:
+            #     pdb.set_trace()
 
             for keyword in self.keywordsToSearch:
                 # pdb.set_trace()
-                if (keyword in description.lower()) or (keyword in content.lower()) :
-                        # pdb.set_trace()
+                if  ((description != None and content != None) and ((keyword in description.lower()) or (keyword in content.lower()))) :
+                # pdb.set_trace()
                 # if "model s" in content.lower() or "model 3" in content.lower() or "model x" in content.lower() or "model y" in content.lower() or "solar" in content.lower() or "battery" in content.lower() or "gigafactory" in content.lower() or "autonomous driving" in content.lower() or "cybertruck" in content.lower() or "elon musk" in content.lower() or "organization change" in content.lower() or "stock price" in content.lower() or "bear market" in content.lower() or "china" in content.lower() or "coronavirus" in content.lower() or "covid-19" in content.lower():
                         lang_code="en"
                         response=paralleldots.sentiment(value['description'], lang_code)

From 46541788ece97ba714ea2aa621f1f02a22cc606a Mon Sep 17 00:00:00 2001
From: Jason Knoespel <jasonknoespel@Jasons-MacBook-Air.local>
Date: Wed, 10 Jun 2020 07:08:14 -0500
Subject: [PATCH 5/5] change filename date format

---
 news_api/.DS_Store                            | Bin 8196 -> 8196 bytes
 news_api/__pycache__/__init__.cpython-37.pyc  | Bin 146 -> 146 bytes
 news_api/__pycache__/items.cpython-37.pyc     | Bin 465 -> 506 bytes
 news_api/__pycache__/pipelines.cpython-37.pyc | Bin 453 -> 2062 bytes
 news_api/__pycache__/settings.cpython-37.pyc  | Bin 323 -> 370 bytes
 news_api/pipelines.py                         |   2 +-
 .../__pycache__/__init__.cpython-37.pyc       | Bin 154 -> 154 bytes
 .../__pycache__/news_agg.cpython-37.pyc       | Bin 1295 -> 2688 bytes
 news_api/spiders/settings.cfg                 |   4 ++++
 9 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 news_api/spiders/settings.cfg

diff --git a/news_api/.DS_Store b/news_api/.DS_Store
index 712f5722b55d7734ab064bba1f83680f1b585f64..86f798c9a58a19033071eae913effe57e4f4029e 100644
GIT binary patch
literal 8196
zcmeHMOK=oL82<kxkeLaP2Eh!=Ym1ml)EG8_@DPHVP1K5jL^mN19?R^`kc>=b;?8U~
zAw=T=EiJB4D0;C<tB{Lg>4^)LWmz7)h@xKf;=xy0rB!;=zq@Bsut~fjRx!QR{rB|$
zfBpF8o9Xlf05DZBTLE+cP$+ZDYN%KsaX-)VQbpLpaY%Ro8v@ACuLZ{EMMn)00}%re
z0}%re0}%uN2L@=*=1Ey(-<RH~j2MU*xGNdp?}s>LZj+&%l~%kus1Qd0vQ^Y964mJ(
zATaUJCPO<bt$<RW^7Mcq6vHD1N;vuBJe+JYw6oGmIG}_Bh9hHmLP0S);o>43Fez<R
zMhrv@%x8e_{WapuZi9Qv`F*O#@p4(m%Tfe?sCH9s`dcKbl~Cg_1vxM%kAlVfDMMW<
zLn&em3&y|~UkS=a!Md{e6%=WMNk51BFVaYPvI}F+lwp_AR|uQ3MOePN#c+R=;cBRV
zi@sz80yVPmXo8OmH}O$TaeO?JuzBzo;Nf4QvqgVA<odJhxMTZ4B5?yNDyx<(tyUCG
z(UxmR)1zK79T-8uPZ{~c{HoUoMzdkP&-0wQx{*C(Tg41p-Q~G~VY^mdfNj&FkRt=O
zYkI|$S8z>V^plnjMNt(s!`4nuZ{69}9B*siKGPhZZf$E%G{<+eCuU|;rLLj1qv!b8
z)T!yyGta$7yivmNl?=7=dak@kwK7&5f1*pSvtqKe&aX@BRJ9uYvB$NZK|ANUY5SCw
zVYNOjvxhEw^VS3{X&~o0g)!I9u$4K($rT(UusR)w2m6D8H5u^Qu<zxAE)kevtl(Qu
zTNGzcSltv&&{e#rgX4kix&9$5?~|=hCgil$%gffpHq|#h)V5<!=k?_)YT3$Fnl?-X
zxq*Gcw(|W&+YCmB3_oYNrtOXlOxV7ib*w(u7_-=KswzPD*6I3SA!|Qdu!AX9uPCB*
z-FjU+V$viIcqZ@6WFIIpKE7Vp4|7jf-lEbB0rxfNTAxo#Vn!Hv{{x!Vcam-(^TVN9
zwrG04V`MFdnW`LT>o!dvn2;-D`@+VZyEJ_;=Z%fgO8k}B95#3C)%9b1<o$*p92~b?
zn$@S_@F|+r5chSOCZCg1pKOveiPfr%@7il&Gi-xB&<jJP{UV%&b8sHsgv;<DT!l~I
zbNB+jgs<QRd=Edu&+rTU3V&iHF2SW(jSSc02Hc2^*o-Z>1rxXn_hJ$|F@^i^DNJJ@
z_Tw=;fum^SGw9=Kd>&`<1$+fx#f$hlzJYJ!yZ9bn!T0ehUc-;%3@b{id_rEAO8l5=
z^Hmu>eHBL+Qe})(*?31)`OPvZ#d}t-*;wDy+`4^ta_(N1-Mw1`cC)60kfc(Cf=UVz
z_wqU^c5Q5(wxOiwa-H{+=B!DD3x`$9=y+_iLRw*pxYQbAjVcM6q4YMy9#j-oA$={e
zgi0D_mD0CWYgb6EY>D*l)E-tz&#X#%JG4g>k}X>*y(z6*r3Z^ax%?It-zB&L*GPEZ
z!gufk`~<(lUkIq+3e-tzn{YGMV}p=eE4JY_><A^-h5PXUK8d|Jgu{3Qk77p1?j$<s
zVi70t6i(v|zKAd3%Ot?Fcn;6wg;0oZ;X7s0>nVv&TD&}8dK2lq>v--6c}jx46?7Y~
zDrMg~{~O@Z#lQdG#u-FiM+`&^+-U}|syp4CqFF5V2I6P!IOXG%dE#-i(h3L_JcPH4
k<2c#5e;86d9{Q4@ot2h6l>YM%0nz?%Inn-q*YE%T0HwvHSpWb4

literal 8196
zcmeHMTWl0n82<llfiu%Wr?qTbWMQQxt*~fYXv<CAUbxgtAhe||tt_)UM>=tKX4#o-
zDYaTJL>~yoOQMmODDlOBFC_Y)K58WDi;+as=!*|N_@coVpYT6(W-Dx46CV&0=WNdT
z&z%4N&gGkL=JZSf0DB5X8$dk(Fv`Me6;v#egkR(ZDH5~@91<SD1P@a5y9aU$qQi!e
zfslcafslcafsld!0t2*Xi^P`L_oX*1Lk2<y{+A4h_d}eru*tyAN$bBlsPK;fWXq}9
zPgJLKfRBj>HW}DCX?-a5DNhd=KruLCpoEh@F2czs13M?Jgab-AU@$TUClvHYCtTbQ
z2TV#EmLUTn0}B}tdw+$0W;emU>HI$3Z#kK?<)kS>JXE_Vw_NuV)rzPbOhX2A$`imv
zKP9MlOHj-ogF_Zv|0_YsC|Fk#zltJFFz9Dd|5G$lp6ue-(<RtS`U+uE=7bfiTMBor
z1Xn@*bNZ6u`KaTIkLL67`VD-npg1v}LD)RFi}3I_(dj8S7jXS~I%k=#*V_6GM#{>U
zEnmTy%G8zWqXQGp)PSdZ1-D1fkBO^;x;K#y>VuAB&DZtxVUtg#wAx<B_H@(cc^_;V
zoI=KjP1|s$dYpo7xPCta(!rR*l$6#mGqZJjdm`T6(K(xl&$P97v?k(R9j&vo3ai`L
zc3=Oo?DVObCuW~~gLtEa!7CZ6<n?@cj%p>WSng!6TxVpuxX!D^bt-B_bnm|XYD%k_
zr0E~Xd8T8#hk4#5o6@RWT5^BJu@C0C@DFDktB|$blvbV5txUnvJ>G3u1Li62j(Gf}
zCu&DrC-3$8fg>hiMO#YK3NAmy31Ck0K7zR#Di<nOuUX&NvZcLiclVW*tEx1$My(y5
zG+i@o@u4Zx@FqreH^Xhiw2vPl#%#}g%;b5cI&w)FwDm0aQPH|+1JfGWNFi;WD45=~
z>}zdwoua+Z#tj<n5yucnL&mR-ZeVOowD;yYm9kmg7;RQGoejD)VcEqdHKAy(GWa;%
zJ_)^7ZDX3D3|V@bTe4}J+R3zsl;KIqnrz&m?qb!LP)0INHcRt%W%K>fB-1Xlqhjob
zb=NzP<Mv;(>SJtFc)AUP8>}(eBJZoe(#B+~yi#b}ZlDJ_3HxE3gfCtPXW%Tn2<PA(
zxCocvWB3d%!x!)+d<EaZ_wWP!2tUKG@Ecr%-w{#83arF>ybCvA6K=#!xD~fyCwAd(
z+=F{@KlbAR9Klf>!*Mh)i#DFXJkH>Acp9I_7w|QF3*W}`coE;j5AY-WL{2SIl;wl+
zx>(|;R9h&^;PHzbT}+m-J$v`vCRu)4DYRH!b4Ts!b&buNTRV0>Fn=3M?%Yk{x=~TS
zh@?t<^%T`1RU56VZ&-7u8XF>A@sp)WFFpa<3ZWo&MMSTU$6}h0j=WTwVk&7{@qH~#
z<ckEJ#1`_Dk&LCUtzC_2%Sf>j*V(12S~*EtdUx$6?{X5a^d`H>3*zBY{<-?j!v*+|
zl=n4UCB6LwzmVeofa|yntFR6ma4p7g9X8|LxET|;h4i)^ci>L!^)<H-58@#l#34M4
zDbz7Ty5pp~9O=$Q4`=Zyd>Ws@XYmZ4#h379d<9>{H%WsRO0?HsRGqYVaiR7m2lBS%
z*vH9JRP4>5TXt0@d#)FM1MFG)`~NK)KkPVUAY|b7Gl1oN1ARR-kEPx};;bE~e2lV4
tB5qDvA3}u)5$*nQob3ER45=Oue96GhNlP9||N4i3aQ`>L{Xg9Q{{-OIrx*YL

diff --git a/news_api/__pycache__/__init__.cpython-37.pyc b/news_api/__pycache__/__init__.cpython-37.pyc
index 1456772003203d6aff9c5d0e9fe0fb0f5bda6dbc..7e4c28aa011fb54126e0d1dd6243d3831d273982 100644
GIT binary patch
delta 20
acmbQlIEj(RiI<m)0SI{c?!-;xX#)Ty9Rv{o

delta 20
acmbQlIEj(RiI<m)0SLO)PsL8;X#)T##{`uC

diff --git a/news_api/__pycache__/items.cpython-37.pyc b/news_api/__pycache__/items.cpython-37.pyc
index cd1b14a028c0dad40f366757f3e2de3feed59bb1..ec378ea024c0008e420af3ed1e9373a411001c8e 100644
GIT binary patch
delta 187
zcmcb}{EL~-iI<m)0SJuw?!@UZP2`JU44Ihjtd`2WfMp>=6c3c<O<@dX&}6#B?U!0!
z>{yWLS(2LTr^z?*fGy*#iT@-`ZgCfsCgo%nXQZYimfT`b&d)1J%`3UZSqvmIb3sgw
z?9|Hg{GycNl?+9KK>LaWCz~<4bBO?%94s7+9E_8j8Pzl(GGJ*CAp|6DaoFVMr<CTT
O+HnAR#Y{kggBburc`|DN

delta 147
zcmeyxe36;YiI<m)0SI_%qvGB(PUMSVbeWj$ESAC;%%I71i`y@?yx6fI)3YQs*H4pY
z;ssmAsL6bclJZg91*J(jnZ+5YDTyVw*pu_~N>cMmRx%Xv0?jJoogB^R&L#k4a<EKZ
i!>FbK5d}+u2tFWji^C>2KczG$)s6$mD`o-`9LxZIxFf&-

diff --git a/news_api/__pycache__/pipelines.cpython-37.pyc b/news_api/__pycache__/pipelines.cpython-37.pyc
index 4a2a1db0b3953f177ffa534eb5520e72ec72ac2a..83342042d712856cf8b35177c6024f1790126bb2 100644
GIT binary patch
literal 2062
zcmaJ?PjeeJ6qmHS_S)VgE~K>3P{J_j;OStWIJDD3nuO_Orgb_%Vb}vZo0VeL**~r%
zH#p;4dhLlX&>Z<Jd;^`B;nc6diTAX2Vu!-+=;=v%E4}x7KfT>qS!oh@e*5FkpW`|q
zf8k*MxS(u9Q->fp;WQ&5_AH}03mGMXosrOi87FgdFZ5_-RSRn{=VtZ19@cX|^eK5v
zxW{X+2(O98nG-f)T<1QFeLHS(`W@*s-oiPg-=SuGAapd6<xnI#QbOyr7$}(c_hdQR
z*COvQ<MUVxo#w(cP;!y!IKd6(4+CWznpy`TNJu#e8K<1RLZ=-#&^2p)aiSiL(r4*N
zWN9H12W;`do<iG&rVxlJdPw`fla6CtC9)&qrr?OFsZq*>lmH&F;_nB&=SoP`8^)?E
zj*C(#*t?gMT=WL1?iFxi6pzwNWi&fnbw?8uK-h^;D#G30!_zSiLQglYxY}Jb>_aSm
z6cuqUqR2F(C@=XqLp_M17vngqR%9^u^3q?~!nqoVYtZazKDFZ|G%>zyH249;C7Qx{
z<3jV!iKubLo!9gxr2d@UAYr{qc?c=4l6^+QCU5X2;NRjc9zY-P6}}4n3SR?w+h*-)
zSq#d(oq0Y&AK`j57({O|8$bb<#}jp0+xlrM-{J{o1|RGZC!4MhK$+4hiO7_Jq*DhZ
z<HRNq($~oGkZ3GZ<5wy_H0?@D$}9uYrDf4^r4Jj)_fa5)<aHDmdA7JA=xLGaDEbH%
z6{1LJ4c^n6S0d@o4HKJBu)`idus+`bWeQ#o$s5Y)Ih|?jLUYb3cV8oGy?)qudeDD-
z5Iz5PKlCegZzpV5+U~)VC*SOTzkks0I9Hs<TQk2+#(_Hq6HbyTQ%_`kk}a~hiY_>k
zWgg8IKLHmNLP9)7oA91)T#a$jd?`)`zOr$C4vL<~2w9nuAsb?TT+pz29|8^$6cC7%
z^-qs%TKM5zC12j}^?0h1@}-c1XQ{rwG}_M6V{uewY5B$RJ^iBS&c5sxLie;#`fDk)
zoct)^VyTB(j>Vmmhj~0KVPev$oBC*CzX)44s@*IVg*LUs_F;X-L&Jb8o-Tofrnx{C
zR*?mZ=~arf%8@9ds!ZI4^@=5KfC%2QXI=8WyI{Kn;03u?VF!Q-AUuZxH77NbAH%%y
zlB`ss^FPi6)?(((m*|b9Bl~wQ7Y&!nBHlHd%i9;OXf~y1z*<T))C1%%tqLY&@cOe-
z>CKpP!8dieISaVs$xZm&)Y1ZqrG{&rCfd}qaxj2eZ#*sz#{(d+y@Cy}l<$Dnma}`X
zSV8Kw*}F)&TqjrM3hst5$~6=i4qOW@kL=IHDasS+;R5nqVl6SPB#V{Gh5n`F@)k~i
zisCaA|5;Jnu!<E!p@q+GJH8hLe$#g)u5<#q0i9V_aJxp6?pUW8)c@+2Cow1T2CiS5
zt&3k1F~YK>B;F;7^}@PteKoc65dW37Y}@@8NN6h-7J1`8sD2Ay;gmuMd=@~l3LKxd
HS=;>waHH)5

delta 170
zcmeAZILgfD#LLUY00h%ryyFxZC-O;1i2!--3@HpLj4eQ%%FxUl#hAhr%%I6U@q9HK
z6HvB@X>tIgKO@)V1&jeH>|g;@f(^uF1QMJ;Tny5}!3gI2X)@hn%TLNmO)dfRiy+2<
Zl!I9yjkh>#a`RJ4b5iX<x<3Ok0|1tx97F&B

diff --git a/news_api/__pycache__/settings.cpython-37.pyc b/news_api/__pycache__/settings.cpython-37.pyc
index b50ae65ce454e2cb4bfdf0c04d047c4a3eb489d3..0b27a6dbaa61b88ae8854ca7e19bd11b7529340f 100644
GIT binary patch
delta 165
zcmX@i^ohyGiI<m)0SKZ_K8W+?VPJR+;=lkikl_Ht#R@<og&~R|g)yBmiZO*TiYbLD
ziaCWjg(Zb`4pS6M3R^IPCi_G$S)P|EK&3B@fP^N~#B>#vDy6*C^5Xc!f=s=F%!1UM
t%)HcMJ>UGi^n4em0I0yk2|;WeKpjOK6JPlAft<s_#Ky?N&Z5V}2mtHoBsu^9

delta 118
zcmeywbePG@iI<m)0SGkeqT-}@7#JRdI55BjWH<nEu^f;{VTfW#VN7R?VoYI-VoG6(
zVoqUBVaa5SVo6~QX3%7t=p#F^RE0HDhmnC{;@%)OHlUs&w#lN5{yZE&CJPfABMUo=
F9socM5y=1m

diff --git a/news_api/pipelines.py b/news_api/pipelines.py
index d30c6e7..b224dba 100644
--- a/news_api/pipelines.py
+++ b/news_api/pipelines.py
@@ -20,7 +20,7 @@ def process_item(self, item, spider):
 
 class MongoDBPipeline(object):
     # pdb.set_trace()
-    today = 'newsAgg_' + datetime.strftime((datetime.now() - timedelta(days=1)), "%Y/%m/%d")
+    today = 'newsAgg_' + datetime.strftime((datetime.now() - timedelta(days=1)), "%Y%m%d")
 
 
     def __init__(self, mongo_uri, mongo_db, mongo_collection):
diff --git a/news_api/spiders/__pycache__/__init__.cpython-37.pyc b/news_api/spiders/__pycache__/__init__.cpython-37.pyc
index 1a918889317817608ae985d6bc49b80fd583467a..2d919ac7bc0e844c10e966475b74f79f41feec94 100644
GIT binary patch
delta 20
acmbQmIE#_TiI<m)0SI{c?!-;x=>q^JRRk#j

delta 20
acmbQmIE#_TiI<m)0SLO)PsL8;=>q^M{{*c7

diff --git a/news_api/spiders/__pycache__/news_agg.cpython-37.pyc b/news_api/spiders/__pycache__/news_agg.cpython-37.pyc
index 2353b3aac6fe06accb1a10444118162ea685d2b9..fb79302fe7549c8a527cad118e2d746d8849d2af 100644
GIT binary patch
literal 2688
zcmai0-E!MD76wR4A|+XIl5QP)vn`t@-l(-C|HXDT^|p@PpG+E0Q#<u8XSy1QfGJx1
zGXTjJN4`pKGm{6{3;VK<vai6|i_Yv-_YwAL4=CA9Iy<xE0QhhK0te@O=YWr=rpgE&
z=N~`+?Vr~W`j_flJ_dj<pokGv6j2-?XWU{3=Y7r5pw$8t>W+@pm>w8m!71d}2#R6J
zDdiXkrenf*A+SQrvBI)bp2#asg;mb0w0MCr`Wn#^HD4iWGIXTB)SW4c_fXyX1vEgr
zb(|IV5}z^-Xlj?8h^@rm4ty3#5L70DCz6GAEi);RO!^_qELBq$NaB3Dv*Nhl?U4s*
z=p8%{R`<U5!sle^sJr#U!)?!B`fmC0i|1Qk{pFjTr%zn9=jAgE;0q|?6;uqV^TXM(
zqf?FQ)PT1@i%NHD(K4;j>fbAl!3uPW)?Q<$$V%rX6zc-9GQCEpU*lVdUcb<s3cW#Z
z()Z}^==*er-lDT~j{ctBrhlLx&=09iKk}xXD!oHLeucmmcj-N7YjhrL^2aL+j7{Q{
zA}|Q?ytLe7ENPIyAF$L4$!UXlY;!TgODTUAr+yHS)`RAvegD9ZXnZ2<-F<s$vAOuj
z2Ep3uBl~o1wQg@ENx%+R=czAS4_4NjD{J=sr{C;9`*y(&{2sHvW^OO8+uL0phpe@}
z(Ohh<JX~CDZmipT<dATG@>QyRzLX9mJC&`j41)y%>-sK{ejK$<mE_aY-%7*a(edVD
z^WlOYg66Fg)=4IiB$2o9N$Zn*eB+(9gzrU+HrT1#C6UJ-4K_O~X+@BbHDd02e&m=D
z8TcNMBE9aVezIUwb{If+!S3+1#-hfHy#*GHx2{*R@^&0W%vFk}Q`^MtvIa;bj{`^F
z+1*cPUnCwUlr^42g1LRp8sD?yJ`+-;<rjkS#+C=XOly10ZG6R~+ifUmR@`!3mb@e1
z^FgP#<~@rkORtURHNK48nEH`-CHNMO*9W^e_VVHC23uQRV{4tx>IxWri9T3lYvdv9
zu)e;$QO8b+B!0WchQDB-EyRdH$xx)`y}!2Zg{^xuErqaGV2ewzatTf^!O$s)I3V09
zb%>OV51on!Ry-uGjQKFVLHaU|;xO(Dn{s~unmT27*kN4qzS~PJ7Q~Sq_C*h@P9i_e
zJ#UYl1pbk++p`nyyDY7A7~ytEc#lb^;CB6pIF=hLukk)8?ZyM2HkKZ`>MZKfp%kI0
zKoyUnqIiUk&<tdx88kuq4D#?9KD{%-JLt!`Gfm<nZKT1d4h7=&wf+Ob=-57h>;+L(
z*iD;VDU(07T6s7nelzA?Yp~p6130_{t-Q~VHzgARnLh;XZ>Gk)j5pIS=S9rr<KZS}
z0UMCW9nJ?6Pi?Yj9{li>4LAR<%k)g|^B^-_5(FLM_Oeo!0m-<i7qWtk!F^d#xSS-z
ztTf?UR!9g3YC-x&bv-i#3l1{_RuOy(L_Z<^J~aQ?ZYlnW))5hL)Qe&!5*D<8T&(3m
zjH}bOLCar($TO{&2t2O|d)9;`T(<dy=^w(vLb(9xb9x!4w<jrhoROOEPIfgl5uOT^
zE5F?VZ%TBIN8pzWe5Sp@r?nAe)>*WJ-ryh6nLdIXJB!W?Xmw}{r~g*o18@F(RvZ;5
z>gjJVuZ@f=1k!Wk0`-cbJ1UNhF;;V^aaNLr^Wu1=QRzZE!{gP@OSAyHn=opQOhA^j
zUZS%yj236ns7y<rA;9J=qE_CwuKMM?U(WlL99MD-y25vQj;lGY<`~Wbat$zsb_!Yx
z+8VUwoQZ1^Zits~7ap%2pyOK;UQX{iCRpBg117Q(M1_Rpl2s@JGWdx~Qkll0RBPH!
zCFyqpUvwGWl5cg01m?tj?y^ktX=X%_8l579Z#U*SQ8G)wns8OZ!Y1U5lbBPH)}k0<
z64IA_822LziP1O$0%e6DCRAibC#J(}`f~byyaz$-cGd6+WV<R*6PU~b8cV?yU9y6J
z3qU$GNGEPr<)(Zeu6{Ov$=rv#yu#m8J(B}RW5}-j<9u*Hf<EK3fccyflqMfLmWs5t
zqOxP<tM9{g!)=%YNgsHv{s{6Q7*{iDg_}^_hawiBLb$5k&`fNWs#;Sw@HDRBvQgBk
zfTnSE^1`@o8tKh<NuCGaZv9$jw%f|4?RHjfw;`|g1BI*Y_HmyC;}QM=?3D*9SIH|I
zU)hPz3#w-d$q(~C7CnkjxC#-j{yX?xRozmR()z8FsKDm=aY}#`5H+Yw&A_*D8PDOn
znrU%0R<H5v(D09-%JhVGxQa&JQk7E0sa+<3Cfr7$82iIe+T_ZOwV78lD}qUtDU|Q>
hwf~nwnfck+1b<f9S}1>l`LGElRj-MsHB&EJ{{`>4@d5w<

literal 1295
zcmZuxPjBQj6t^8G@lL1Z&#qP!I3RHv#CFQCw2M@sg<T23VGgVS$zq8-v72V<NmAQc
zm@0GXUdspIw0q<mAzwM|SK!2R(o$B!(R=>cao(Tbd(Kze+am<y>F@mKKN&)Qg~j!t
z03L$aZ$U{Uu||BDF~_hb8m$N?IP66_T2W4WOm)0Uc!JT_NX9aGj%1?Hl3YYQmG}vo
zq_4mpnoqD#o;0OY2GI6g{bIjt%163Xm4l6~tBr4*TIJ#V^@spG1hF51QYb{iiNrio
zR1z6oVjfE><L8JcGLb3Fsbt{w*1tXmr#!8|r6>yi!R+|{19dPvPzQ^}{ZGZQ_(XoT
zuMWiLd9Dr*XAdTrr=lsdQ?<@9cnyPqq#z+S`!T2vEzuH73_(B`A%x3#gB@8$9fo&Q
zCYR&~6bK3w%z5mT)#7|K+n?><+n?Q=9eScFK5FWw)xwl+J&8R&^P^)^ud+rMvGT;#
zo@r4Pttb>vZEak(SWk$LTBAM7h1Ls^pTdVja!tv13gt5OOqsPiDXYRKHaDVK`}8|?
z)++0K45qCD8`LU&%*S9O6ULRfR`w5M-iFlvel-2wDr2WhVe9I&s+Db&p60bw)1q|K
zKx764-2mIy)n08B#U6Ovu@3BJX7Fp@2*iy+A;uzt!^5KU_pi!6)M)SZPx3%|7zs$|
z&Yt|=066plcX&x&;tlz=>!2TY(TnIMdW@dxpU5WKP)9G&ChnpxmgER^R7Sf9aMWYk
zW7^}m$8nF79w$9cdz|)|^_U$Y7kBXm#$YLw_Uzpshw|Q<b9T)k-)h>0F71hu!g*3k
zPb;xfJQ1xssg2)~3QAHouB<Dw1-bY%uPYaBLE`BW2=}qBg|wb7YPt4t2{*!;kUrGT
zl;NCtE3BA-y3`d^>b<axpk00ZOz0L`<%-t6oP)XzXUv<RJ_NDvfI@go?&1V9Jid7d
z!{@hOFG}Aq^9l1T%R(wy=0{n!s%5JK9%tEEEA;Tj+y%QP+*z|7)SaM$*Z)KkTFms}
z#DK#8$ilA(F-*ybusbH~PMFz&$?OId+F&pj4D+4q{@c5urvK>we0X*=MEXUz$u@Xn
Jn2|nX`VWTtLY)8r

diff --git a/news_api/spiders/settings.cfg b/news_api/spiders/settings.cfg
new file mode 100644
index 0000000..99b25bc
--- /dev/null
+++ b/news_api/spiders/settings.cfg
@@ -0,0 +1,4 @@
+[PD]
+exist = true
+api_key = D3ZcPSa5zmgWQl4SRgmQa1jhAV9Cgi1P2BUQAFXHDKI
+