Pulling fetching out of Feed

Makefile

@@ -4,7 +4,7 @@ coverage:
 					 GOOGLE_OAUTH_SECRET="test" \
 					 GOOGLE_OAUTH_ID="test" \
 					 GOOGLE_OAUTH_FQDN="test" \
-					 APP_FQDN="rssmtp.firetaco.com" \
+					 APP_FQDN="testing-fqdn.rssmtp.example.com" \
 					 APP_SECRET="test" \
 					 APP_SMTP_HOST="smtp.example.com" \
 					 APP_SMTP_PORT="465" \
@@ -29,7 +29,7 @@ test:
 					 GOOGLE_OAUTH_SECRET="test" \
 					 GOOGLE_OAUTH_ID="test" \
 					 GOOGLE_OAUTH_FQDN="test" \
-					 APP_FQDN="rssmtp.firetaco.com" \
+					 APP_FQDN="testing-fqdn.rssmtp.example.com" \
 					 APP_SECRET="test" \
 					 APP_SMTP_HOST="smtp.example.com" \
 					 APP_SMTP_PORT="465" \
@@ -45,13 +45,9 @@ install:
 	# for node.js
 	sudo apt-get -y install python-software-properties
 	sudo add-apt-repository -y ppa:chris-lea/node.js
-	# for mongodb
-	sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10
-	sudo sh -c 'echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" > /etc/apt/sources.list.d/10gen.list'
 	# install packages
 	sudo apt-get update
-	sudo apt-get install -y build-essential mongodb-10gen nodejs
+	sudo apt-get install -y build-essential nodejs
 	npm install
-	npm install mongodb --mongodb:native
 
 .PHONY: cov coverage dev supper test testwatch

models/Feed.js

@@ -138,6 +138,15 @@ function init(Sequelize, sequelize, name, models) {
     this.save().done(done);
   };
 
+  methods.asRequestOptions = function() {
+    return {
+      url: this.url
+      , headers: {
+        'Last-Modified': moment(this.last_fetched).format("ddd, DD MMM YYYY HH:mm:ss ZZ")
+      }
+    }
+  };
+
   var Klass = sequelize.define(
     name
     , attrs

models/fetcher.js

@@ -0,0 +1,27 @@
+var request = require('request')
+;
+
+function Fetcher() {
+}
+
+Fetcher.prototype.updateFeed = function(feed, done) {
+  var options = feed.asRequestOptions()
+    , agent   = [
+      'RSSMTP (https://github.com/elliotf/rssmtp; '
+      , 'http://', process.env.APP_FQDN
+      , ')'
+    ].join('');
+  ;
+
+  options = {
+    headers: {
+      'User-Agent': ['RSSMTP (https://github.com/elliotf/rssmtp; http://', process.env.APP_FQDN, ')'].join('')
+    }
+  };
+
+  request.get(options, function(err, whatever){
+    done();
+  });
+};
+
+module.exports = Fetcher;

test/models/Article.js

@@ -238,7 +238,7 @@ describe("Article model (RDBMS)", function() {
           "<h1><a href=\"http://example.com/an_article\">article title here: with &lt;brackets &amp; such&gt;</a></h1>",
           "<p>article content here, with <brackets> and &'s</p>",
           "<br><br>",
-          "<a href=\"http://rssmtp.firetaco.com/feed/", this.feed.id, "\">unsubscribe</a>"
+          "<a href=\"http://testing-fqdn.rssmtp.example.com/feed/", this.feed.id, "\">unsubscribe</a>"
         ].join('');
 
         expect(emailData.bcc).to.have.length(2);
@@ -251,9 +251,9 @@ describe("Article model (RDBMS)", function() {
           , subject: "article title here: with <brackets & such>"
           , date:  this.data.date
           , headers: {
-            "List-ID": this.feed.id + '.rssmtp.firetaco.com'
-            , "List-Unsubscribe": 'http://rssmtp.firetaco.com/feed/' + this.feed.id
-            , "List-Subscribe": 'http://rssmtp.firetaco.com/feed/' + this.feed.id
+            "List-ID": this.feed.id + '.testing-fqdn.rssmtp.example.com'
+            , "List-Unsubscribe": 'http://testing-fqdn.rssmtp.example.com/feed/' + this.feed.id
+            , "List-Subscribe": 'http://testing-fqdn.rssmtp.example.com/feed/' + this.feed.id
           }
           , html: expectedHTML
           , generateTextFromHTML: true

test/models/Feed.js

@@ -427,6 +427,19 @@ describe("Feed model (RDBMS)", function() {
         });
       });
     });
+
+    describe("#asRequestOptions", function() {
+      it("returns a `request` ready options object", function() {
+        var options = this.feed.asRequestOptions();
+
+        expect(options).to.have.keys(['url', 'headers']);
+        expect(options.url).to.equal('http://example.com/feed_methods');
+        expect(options.headers).to.have.keys(['Last-Modified']);
+
+        // This assertion is stupid.  The reason for this stupidity is due to timezone differences.
+        expect(options.headers['Last-Modified']).to.match(/^\w{3}, \d{2} \w{3} \d{4} \d\d:\d\d:\d\d [-+]?\d{4}$/);
+      });
+    });
   });
 });
 

test/models/fetcher.js

@@ -0,0 +1,64 @@
+var models  = require('../../models')
+  , expect  = require('chai').expect
+  , request = require('request')
+;
+
+describe("Fetcher model", function() {
+  beforeEach(function() {
+    this.fetcher = new models.fetcher();
+
+    this.fakeHttpErr      = null; //{fake: 'HttpErr'};
+    this.fakeHttpResponse = {fake: 'HttpResponse'};
+    this.fakeHttpBody     = [
+      '<?xml version="1.0" encoding="utf-8"?>'
+      , '<feed xmlns="http://www.w3.org/2005/Atom">'
+        , '<title>a title</title>'
+      , '</feed>'
+    ].join('');
+
+    this.sinon.stub(request, 'get', function(args, done){
+      done(this.fakeHttpErr, this.fakeHttpResponse, this.fakeHttpBody);
+    }.bind(this));
+  });
+
+  it("can be instantiated", function() {
+    expect(this.fetcher).to.be.an.instanceof(models.fetcher);
+  });
+
+  describe("#updateFeed", function() {
+    beforeEach(function() {
+      this.feed = models.Feed.build({
+        name: 'Feed methods feed'
+        , url: 'http://example.com/feed_methods'
+      });
+
+      this.sinon.stub(this.feed, 'asRequestOptions', function() {
+        return {
+          fake: 'requestOptions'
+          , url: 'http://fakeRequestOptions.example.com'
+          , headers: {
+            'Last-Modified': 'fakeLastModified'
+          }
+        }
+      });
+    });
+
+    it("adds a user agent to the request options", function(done) {
+      this.fetcher.updateFeed(this.feed, function(err, meta, articles){
+        expect(err).to.not.exist;
+
+        expect(this.feed.asRequestOptions).to.have.been.called;
+        expect(request.get).to.have.been.called;
+
+        var options = request.get.args[0][0];
+
+        expect(options.headers['User-Agent']).to.exist;
+        expect(options.headers['User-Agent']).to.contain('RSSMTP');
+        expect(options.headers['User-Agent']).to.contain('https://github.com/elliotf/rssmtp');
+        expect(options.headers['User-Agent']).to.contain('http://testing-fqdn.rssmtp.example.com');
+
+        done();
+      }.bind(this));
+    });
+  });
+});