From ac7b1dda6155d80c4394af7fe04273e4bd4910af Mon Sep 17 00:00:00 2001 From: Calvin Jeong Date: Sun, 21 Sep 2014 22:41:59 +1200 Subject: [PATCH 1/3] Use generated URLs rather than constant URLs. Fixed URLs do not work well when the app is mounted under a subdirectory. --- earthreader/web/templates/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthreader/web/templates/index.html b/earthreader/web/templates/index.html index e5e97bd..5e89592 100644 --- a/earthreader/web/templates/index.html +++ b/earthreader/web/templates/index.html @@ -54,14 +54,14 @@

Earth Reader

  • Add Feed

    -
    +

    Add Category

    -
    +
    From a2cb850f9e179c0a2a52ecbfea745e3e321b1176 Mon Sep 17 00:00:00 2001 From: Calvin Jeong Date: Sat, 27 Sep 2014 15:33:54 +1200 Subject: [PATCH 2/3] Proivde `User-Agent` header Some sites expect to have such header (ie.https://news.ycombinator.com/rss) --- earthreader/web/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/earthreader/web/__init__.py b/earthreader/web/__init__.py index 3f4514f..529faec 100644 --- a/earthreader/web/__init__.py +++ b/earthreader/web/__init__.py @@ -7,6 +7,7 @@ from six.moves import urllib from flask import Flask, jsonify, render_template, request, url_for +from libearth import version from libearth.codecs import Rfc3339 from libearth.compat import text_type from libearth.crawler import crawl @@ -166,9 +167,11 @@ def add_feed(category_id): cursor = Cursor(category_id) url = request.form['url'] try: - f = urllib.request.urlopen(url) - document = f.read() - f.close() + rq = urllib.request.Request(url) + rq.add_header('User-Agent', '{0}/{1}'.format(version.__package__, + version.VERSION)) + with urllib.request.urlopen(rq) as f: + document = f.read() except Exception: r = jsonify( error='unreachable-url', From 965afc9723b0cd6f73c45eaae204cc7f3c24ccc5 Mon Sep 17 00:00:00 2001 From: Calvin Jeong Date: Sat, 27 Sep 2014 19:20:36 +1200 Subject: [PATCH 3/3] addinfourl instance has no attribute '__exit__' --- earthreader/web/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/earthreader/web/__init__.py b/earthreader/web/__init__.py index 529faec..84a095f 100644 --- a/earthreader/web/__init__.py +++ b/earthreader/web/__init__.py @@ -170,8 +170,9 @@ def add_feed(category_id): rq = urllib.request.Request(url) rq.add_header('User-Agent', '{0}/{1}'.format(version.__package__, version.VERSION)) - with urllib.request.urlopen(rq) as f: - document = f.read() + f = urllib.request.urlopen(rq): + document = f.read() + f.close() except Exception: r = jsonify( error='unreachable-url',