Allow custom RSS processing of non-standard feeds

docs/README.md

@@ -2,6 +2,8 @@
 
 Converts RSS feeds to `JSON` (suitable for rendering with e.g. the RSS [widget type][angularjs-portal widgets docs] in [AngularJS-portal][]).
 
+There is also the functionality to plug in a custom XML for feeds which are not in a standard rss format.
+
 Intended for deployment as a "microservice".
 
 ## Confidence-inspiring badges
@@ -80,6 +82,13 @@ returns something like
   ]
 }
 ```
+##Custom Filters /{webapp-name}/rssTransform/custom/{key}
+
+Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide.
+
+As in the standard rss processor, {key} is the string identifying your feed. To utilize the custom filter, create a class which implements the iFilter interface.
+
+Give your class the case-insensitive name of your feed, plus the word "filter".. i.e. if your endpoint is sports=http://www.ncaa.com/news/ncaa/d1/rss.xml, then your class would be named SportsFilter.
 
 [AngularJS-portal]: https://github.com/UW-Madison-DoIT/angularjs-portal
 [angularjs-portal widgets docs]: http://uw-madison-doit.github.io/angularjs-portal/latest/#/md/widgets

pom.xml

@@ -65,6 +65,11 @@
             <groupId>org.json</groupId>
             <artifactId>json</artifactId>
         </dependency>
+        <dependency>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>fluent-hc</artifactId>
+          <version>4.5.6</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>

src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java

@@ -13,6 +13,10 @@
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.beans.factory.BeanFactory;
+
+import edu.wisc.my.rssToJson.filter.XmlFilter;
+import edu.wisc.my.rssToJson.filter.iFilter;
 
 import edu.wisc.my.rssToJson.service.RssToJsonService;
 
@@ -26,9 +30,40 @@ public class RssToJsonController {
     public void setRSSToJSONService(RssToJsonService rssToJsonService) {
         this.rssToJsonService = rssToJsonService;
     }
+
+    @RequestMapping(value="/rssTransform/custom/{feed}")
+    public @ResponseBody void customFeedAsJson(HttpServletRequest request, HttpServletResponse response,
+            @PathVariable String feed) {
+
+        JSONObject jsonFromFeed = rssToJsonService.getCustomizedUrl(feed);
+        if (jsonFromFeed == null) {
+            logger.warn("No feed for endpoint {}", feed);
+            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+        } else {
+            /* The filter is a custom class based on the name of your endpoint.
+             *  For example, if your feed is called "sports", then you should
+            // have a class in the filter package called "SportsFilter".
+            // All filter classes should implement the iFilter interface.
+            // 
+            // The static method XmlFilter.getXmlFilter(feed) will use
+            // reflection to return a filter with your custom business logic. 
+            */
+            iFilter filter = XmlFilter.getXmlFilter(feed);
+
+            JSONObject jsonToReturn = filter.getFilteredJSON(jsonFromFeed);
+            response.setContentType("application/json");
+            try {
+                response.getWriter().write(jsonToReturn.toString());
+                response.setStatus(HttpServletResponse.SC_OK);
+            } catch (IOException e) {
+                logger.warn(e.getMessage());
+                response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+            }
+        }
+    }
 
     @RequestMapping(value="/rssTransform/{feed}")
-    public @ResponseBody void getJsonifiedRssUrl(HttpServletRequest request,
+    public @ResponseBody void rssAsJson(HttpServletRequest request,
             HttpServletResponse response, @PathVariable String feed) {
 
         logger.debug("Attempting to retrieve feed for endpoint {}", feed);

src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java

@@ -1,9 +1,11 @@
 package edu.wisc.my.rssToJson.dao;
 
 import com.rometools.rome.feed.synd.SyndFeed;
+import org.json.JSONObject;
 
 public interface RssToJsonDao{
-
+    public JSONObject getXMLFeed(String feedEndpoint);
     public SyndFeed getRssFeed(String feedEndpoint);
+    public String getEndpointURL(String feedEndpoint);
 
 }

src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java

@@ -1,12 +1,23 @@
 package edu.wisc.my.rssToJson.dao;
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
-
-import org.apache.http.HttpHeaders;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.json.JSONObject;
+import org.json.XML;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils; 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,36 +42,81 @@ public class RssToJsonDaoImpl implements RssToJsonDao{
     @Autowired
     void setEnv(Environment env) {
       this.env = env;
+    }   
+    private String httpResponseAsString(String url) throws IOException {
+        logger.error("HTTP Response method " + url);
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        try {
+            HttpGet httpget = new HttpGet(url);
+            logger.error(httpget.toString());
+
+            // Create a custom response handler
+            ResponseHandler<String> responseHandler = new ResponseHandler<String>() {
+
+                @Override
+                public String handleResponse(
+                        final HttpResponse response) throws ClientProtocolException, IOException {
+                     logger.error("TWO " + response.toString());        
+                    int status = response.getStatusLine().getStatusCode();
+                    logger.debug(status + " response code ");
+                    if (status >= 200 && status < 300) {
+                        HttpEntity entity = response.getEntity();
+                        return entity != null ? EntityUtils.toString(entity) : null;
+                    } else {
+                        throw new ClientProtocolException("Unexpected response status: " + status);
+                    }
+                }
+            };
+            String responseBody = httpclient.execute(httpget, responseHandler);
+            logger.error("ONE POINT SIX " + responseBody);
+            return responseBody;
+        } finally {
+            httpclient.close();
+        }
     }
 
+
+    @Override
+    public JSONObject getXMLFeed(String feedEndpoint) {
+       String endpointURL = getEndpointURL(feedEndpoint);
+       JSONObject jsonObject = null;
+       try {
+         String xmlString = httpResponseAsString(endpointURL);
+         jsonObject = XML.toJSONObject(xmlString);
+       } catch (Exception e) {
+           logger.warn(e.getMessage());
+           return null;
+       }
+       return jsonObject;
+   }
+
+   public String getEndpointURL(String feed) {
+       logger.error("GETTING THE ENDPOINT FOR " + feed);
+       String endpointURL = env.getProperty(feed);
+       logger.error(endpointURL);
+       if (endpointURL == null) {
+           logger.warn("No corresponding feed url for requested endpoint {}", feed);
+           return null;
+       }
+       return endpointURL;
+   }
+
     @Override
     @Cacheable(cacheNames="feeds", sync=true)
     public SyndFeed getRssFeed(String feedEndpoint) {
-        logger.info("Fetching feed for {} ", feedEndpoint);
-        //see if property file has corresponding url for requested endpoint
-        String endpointURL = env.getProperty(feedEndpoint);
-        if (endpointURL == null){
-          logger.warn("No corresponding feed url for requested endpoint {}",
-                  feedEndpoint);
-          return null;
-        }
-        SyndFeed feed = null;
-        try{
-            HttpClient client = HttpClientBuilder.create().build();
-            HttpGet request = new HttpGet(endpointURL);
-            request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service");
-            request.setHeader(HttpHeaders.CONTENT_ENCODING, "UTF-8");
-            HttpResponse response = client.execute(request);
-            SyndFeedInput input = new SyndFeedInput();
-            feed = input.build(new InputStreamReader(response.getEntity().getContent(), "UTF-8"));
-            feed.setFeedType("UTF-8");
-            logger.debug("CONTENT OF FEED " + endpointURL);
-            logger.debug(feed.toString());
-
-        }catch(Exception ex){
-            logger.error("Error while fetching xml from {}", endpointURL, ex);
-        }
+      try{  
+        String result = httpResponseAsString(feedEndpoint);
+        SyndFeedInput input = new SyndFeedInput();
+        InputStream stream = new ByteArrayInputStream(result.getBytes("UTF-8"));
+        SyndFeed feed = input.build(new InputStreamReader(stream, "UTF-8"));
+        logger.debug("CONTENT OF FEED " + feedEndpoint);
+        logger.debug(feed.toString());
         return feed;
+      } catch (Exception e) {
+          logger.warn("Could not get feed " + feedEndpoint + " " + e.getMessage());
+      } 
+
+      return null;
     }
 
 }

src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java

@@ -0,0 +1,53 @@
+package edu.wisc.my.rssToJson.filter;
+ import java.util.Iterator;
+ import org.json.JSONArray;
+import org.json.JSONObject;
+ import java.util.ArrayList;
+ import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+ import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+ public class WudFilter implements iFilter{
+    protected final Logger logger = LoggerFactory.getLogger(getClass());
+     public WudFilter(){
+    }
+     public JSONObject getFilteredJSON(JSONObject rawJSON){
+        ObjectMapper om = new ObjectMapper();
+        JSONObject feedInfo = new JSONObject();
+        JSONObject feed = new JSONObject();
+        JSONArray items = new JSONArray();
+         try{
+            JsonNode rootNode = om.readTree(rawJSON.toString());
+            feedInfo.put("title", rootNode.findValue("title").asText());
+            feedInfo.put("link", "https://union.wisc.edu/events-and-activities/event-calendar/");
+            feedInfo.put("description", rootNode.findValue("description").asText());
+            feedInfo.put("pubDate", rootNode.findValue("lastBuildDate").asText());
+
+
+            JsonNode events = rootNode.findValue("event");
+
+            Iterator<JsonNode> iter = events.elements();
+
+            while(iter.hasNext()){
+                JSONObject item = new JSONObject();
+                JsonNode anEvent = iter.next();
+                item.put("title", anEvent.findValue("event_title").asText());
+                item.put("link", anEvent.findValue("url").asText());
+                item.put("description",anEvent.findValue("short_description").asText());
+                items.put(item);
+            }
+            feed.put("feed", feedInfo);
+            feed.put("items",items);
+
+
+        }catch(Exception e){
+            logger.error(e.getMessage());
+        };
+         feed.put("status", "ok");
+         return feed;
+    }
+     public String healthCheck(){
+        return "WudFilter health check";
+    }     
+} 

src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java

@@ -0,0 +1,35 @@
+package edu.wisc.my.rssToJson.filter;
+ import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+ public class XmlFilter {
+     protected final Logger logger = LoggerFactory.getLogger(getClass());
+     public static iFilter getXmlFilter(String filterName){
+        try{ 
+          String filterClass = XmlFilter.toTitleCase(filterName) + "Filter";
+          String pkg = new CurrentClassGetter().getPackageName();
+          iFilter filter = (iFilter) Class.forName(pkg + "." +filterClass).newInstance();
+          return filter;
+       } catch (Exception e){
+           return null;
+       }
+    }
+    public static class CurrentClassGetter extends SecurityManager {
+    public String getPackageName() {
+      return getClassContext()[1].getPackage().getName(); 
+    }
+  }
+     private static String toTitleCase(String filterNameIn){
+        StringBuilder titleCase = new StringBuilder();
+        boolean nextTitleCase = true;
+        for (char c : filterNameIn.toCharArray()) {
+             if (nextTitleCase) {
+                 c = Character.toTitleCase(c);
+                 nextTitleCase = false;
+             }
+
+             titleCase.append(c);
+         }
+         String filterNameOut = titleCase.toString().trim();
+        return filterNameOut;
+    }
+ }

src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java

@@ -0,0 +1,6 @@
+package edu.wisc.my.rssToJson.filter;
+ import org.json.JSONObject;
+ public interface iFilter{
+    public JSONObject getFilteredJSON(JSONObject rawJSON);
+    public String healthCheck();
+}

src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java

@@ -5,5 +5,6 @@
 
 @Service
 public interface RssToJsonService {
+	public JSONObject getCustomizedUrl(String feed);
 	public JSONObject getJsonFromURL(String url);
 }

src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java

@@ -5,15 +5,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
+import org.springframework.context.annotation.Primary;
 
 import com.rometools.rome.feed.synd.SyndEntry;
 import com.rometools.rome.feed.synd.SyndFeed;
 
 import edu.wisc.my.rssToJson.dao.RssToJsonDao;
 
-
+@Primary
 @Service
 public class RsstoJsonServiceImpl implements RssToJsonService {
     protected final Logger logger = LoggerFactory.getLogger(getClass());
@@ -27,7 +27,8 @@ void setRssToJsonDao(RssToJsonDao rssToJsonDao){
 
     @Override
     public JSONObject getJsonFromURL(String endpoint) {
-        SyndFeed feed = rssToJsonDao.getRssFeed(endpoint);
+        String url = rssToJsonDao.getEndpointURL(endpoint);
+        SyndFeed feed = rssToJsonDao.getRssFeed(url);
         if(feed == null){
             logger.warn("No feed returned for endpoint: {}", endpoint);
             return null;
@@ -58,4 +59,10 @@ public JSONObject getJsonFromURL(String endpoint) {
         stringToClean = stringToClean.replaceAll("\\\\u2014", "-");
         return new JSONObject(stringToClean);
     }
+
+    @Override
+    public JSONObject getCustomizedUrl(String endpoint) {
+        JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(endpoint);
+        return xmlJSONObj;
+    }
 }

src/main/resources/endpoint.properties

@@ -12,4 +12,5 @@ uwp-news=https://www.uwp.edu/explore/media/rss.xml
 working-at-uw=https://working.wisc.edu/feed/
 uw-system-payroll-benefits-news=https://uwservice.wisconsin.edu/news/feed/B
 xkcd=http://xkcd.com/rss.xml
+wud=https://union.wisc.edu/events-and-activities/event-calendar/feed/wud.xml
 sample=https://raw.githubusercontent.com/UW-Madison-DoIT/rssToJson/master/src/main/resources/sampleFile.rss

src/main/resources/logback.xml

@@ -52,7 +52,7 @@
   <root level="ERROR">
       <appender-ref ref="LOG"/>
   </root>
-  <logger name="edu.wisc" additivity="false" level="WARN">
+  <logger name="edu.wisc" additivity="false" level="ERROR">
         <appender-ref ref="LOG"/>
     </logger>