From df53f191ed5f0920869c8034daedd559875ef9e3 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Wed, 21 Nov 2018 12:42:05 -0600 Subject: [PATCH 1/8] feat: support for custom xml handling --- docs/README.md | 9 ++++ pom.xml | 5 ++ .../controller/RssToJsonController.java | 35 ++++++++++++ .../wisc/my/rssToJson/dao/RssToJsonDao.java | 3 +- .../my/rssToJson/dao/RssToJsonDaoImpl.java | 37 +++++++++++++ .../wisc/my/rssToJson/filter/WudFilter.java | 53 +++++++++++++++++++ .../wisc/my/rssToJson/filter/XmlFilter.java | 35 ++++++++++++ .../edu/wisc/my/rssToJson/filter/iFilter.java | 6 +++ .../rssToJson/service/RssToJsonService.java | 1 + .../service/RsstoJsonServiceImpl.java | 10 +++- src/main/resources/endpoint.properties | 1 + 11 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java create mode 100644 src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java create mode 100644 src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java diff --git a/docs/README.md b/docs/README.md index 46e8f7d..741277c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,8 @@ Converts RSS feeds to `JSON` (suitable for rendering with e.g. the RSS [widget type][angularjs-portal widgets docs] in [AngularJS-portal][]). +There is also the functionality to plug in a custom XML for feeds which are not in a standard rss format. + Intended for deployment as a "microservice". ## Confidence-inspiring badges @@ -80,6 +82,13 @@ returns something like ] } ``` +##Custom Filters /{webapp-name}/rssTransform/prop/{key}/xml + +Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide. + +As in the standard rss processor, {key} is the string identifying your feed. To utilize the custom filter, create a class which implements the iFilter interface. + +Give your class the case-insensitive name of your feed, plus the word "filter".. i.e. if your endpoint is sports=http://www.ncaa.com/news/ncaa/d1/rss.xml, then your class would be named SportsFilter. [AngularJS-portal]: https://github.com/UW-Madison-DoIT/angularjs-portal [angularjs-portal widgets docs]: http://uw-madison-doit.github.io/angularjs-portal/latest/#/md/widgets diff --git a/pom.xml b/pom.xml index 63d7609..256582a 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,11 @@ org.json json + + javax.json + javax.json-api + 1.1.4 + org.springframework.boot spring-boot-starter-web diff --git a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java index a74c5be..81ae041 100644 --- a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java +++ b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java @@ -13,6 +13,10 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.beans.factory.BeanFactory; + +import edu.wisc.my.rssToJson.filter.XmlFilter; +import edu.wisc.my.rssToJson.filter.iFilter; import edu.wisc.my.rssToJson.service.RssToJsonService; @@ -26,6 +30,37 @@ public class RssToJsonController { public void setRSSToJSONService(RssToJsonService rssToJsonService) { this.rssToJsonService = rssToJsonService; } + + @RequestMapping(value="/rssTransform/{feed}/xml") + public @ResponseBody void getJsonifiedXMLUrl(HttpServletRequest request, HttpServletResponse response, + @PathVariable String feed) { + + JSONObject jsonFromFeed = rssToJsonService.getJsonifiedXMLUrl(feed); + if (jsonFromFeed == null) { + logger.error("No feed for endpoint {}", feed); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } else { + /* The filter is a custom class based on the name of your endpoint. + * For example, if your feed is called "sports", then you should + // have a class in the filter package called "SportsFilter". + // All filter classes should implement the iFilter interface. + // + // The static method XmlFilter.getXmlFilter(feed) will use + // reflection to return a filter with your custom business logic. + */ + iFilter filter = XmlFilter.getXmlFilter(feed); + + JSONObject jsonToReturn = filter.getFilteredJSON(jsonFromFeed); + response.setContentType("application/json"); + try { + response.getWriter().write(jsonToReturn.toString()); + response.setStatus(HttpServletResponse.SC_OK); + } catch (IOException e) { + logger.warn(e.getMessage()); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + } @RequestMapping(value="/rssTransform/{feed}") public @ResponseBody void getJsonifiedRssUrl(HttpServletRequest request, diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java index f2502af..5418524 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java @@ -1,9 +1,10 @@ package edu.wisc.my.rssToJson.dao; import com.rometools.rome.feed.synd.SyndFeed; +import org.json.JSONObject; public interface RssToJsonDao{ - + public JSONObject getXMLFeed(String feedEndpoint); public SyndFeed getRssFeed(String feedEndpoint); } diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java index b84e805..5c8843a 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java @@ -1,5 +1,6 @@ package edu.wisc.my.rssToJson.dao; +import java.io.InputStream; import java.io.InputStreamReader; import org.apache.http.HttpHeaders; @@ -7,6 +8,9 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; +import org.json.JSONObject; +import org.json.XML; +import javax.json.JsonReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -33,6 +37,39 @@ void setEnv(Environment env) { this.env = env; } + @Override + public JSONObject getXMLFeed(String feedEndpoint) { + String endpointURL = getEndpointURL(feedEndpoint); + JSONObject jsonObject = null; + try { + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(endpointURL); + request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service"); + HttpResponse response = client.execute(request); + InputStreamReader isr = new InputStreamReader(response.getEntity().getContent()); + StringBuffer stringBuffer = new StringBuffer(); + int i; + while ((i = isr.read()) != -1) { + stringBuffer.append((char) i); + } + String xmlString = stringBuffer.toString(); + jsonObject = XML.toJSONObject(xmlString); + } catch (Exception e) { + logger.warn(e.getMessage()); + return null; + } + return jsonObject; + } + + private String getEndpointURL(String feed) { + String endpointURL = getEndpointURL(feed); + if (endpointURL == null) { + logger.warn("No corresponding feed url for requested endpoint {}", feed); + return null; + } + return endpointURL; + } + @Override @Cacheable(cacheNames="feeds", sync=true) public SyndFeed getRssFeed(String feedEndpoint) { diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java new file mode 100644 index 0000000..a79a808 --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java @@ -0,0 +1,53 @@ +package edu.wisc.my.rssToJson.filter; + import java.util.Iterator; + import org.json.JSONArray; +import org.json.JSONObject; + import java.util.ArrayList; + import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + public class WudFilter implements iFilter{ + protected final Logger logger = LoggerFactory.getLogger(getClass()); + public WudFilter(){ + } + public JSONObject getFilteredJSON(JSONObject rawJSON){ + ObjectMapper om = new ObjectMapper(); + JSONObject responseObj = new JSONObject(); + JSONObject feedInfo = new JSONObject(); + try{ + JsonNode rootNode = om.readTree(rawJSON.toString()); + feedInfo.put("title", rootNode.findValue("title").asText()); + feedInfo.put("link", "https://union.wisc.edu/events-and-activities/event-calendar/"); + feedInfo.put("description", rootNode.findValue("description").asText()); + feedInfo.put("pubDate", rootNode.findValue("lastBuildDate").asText()); + + + JsonNode events = rootNode.findValue("event"); + + Iterator iter = events.elements(); + ArrayList eventNodes = new ArrayList(); + + while(iter.hasNext()){ + JSONObject item = new JSONObject(); + JsonNode anEvent = iter.next(); + item.put("title", anEvent.findValue("event_title").asText()); + item.put("link", anEvent.findValue("url").asText()); + item.put("description",anEvent.findValue("short_description").asText()); + JSONObject thisItem = new JSONObject(); + thisItem.put("item", item); + eventNodes.add(thisItem); + } + JSONArray allEvents = new JSONArray(eventNodes); + feedInfo.put("items",allEvents); + + }catch(Exception e){ + logger.error(e.getMessage()); + }; + return feedInfo; + } + public String healthCheck(){ + return "WudFilter health check"; + } +} diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java new file mode 100644 index 0000000..91eb623 --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java @@ -0,0 +1,35 @@ +package edu.wisc.my.rssToJson.filter; + import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class XmlFilter { + protected final Logger logger = LoggerFactory.getLogger(getClass()); + public static iFilter getXmlFilter(String filterName){ + try{ + String filterClass = XmlFilter.toTitleCase(filterName) + "Filter"; + String pkg = new CurrentClassGetter().getPackageName(); + iFilter filter = (iFilter) Class.forName(pkg + "." +filterClass).newInstance(); + return filter; + } catch (Exception e){ + return null; + } + } + public static class CurrentClassGetter extends SecurityManager { + public String getPackageName() { + return getClassContext()[1].getPackage().getName(); + } + } + private static String toTitleCase(String filterNameIn){ + StringBuilder titleCase = new StringBuilder(); + boolean nextTitleCase = true; + for (char c : filterNameIn.toCharArray()) { + if (nextTitleCase) { + c = Character.toTitleCase(c); + nextTitleCase = false; + } + + titleCase.append(c); + } + String filterNameOut = titleCase.toString().trim(); + return filterNameOut; + } + } diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java new file mode 100644 index 0000000..fcbf8ba --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java @@ -0,0 +1,6 @@ +package edu.wisc.my.rssToJson.filter; + import org.json.JSONObject; + public interface iFilter{ + public JSONObject getFilteredJSON(JSONObject rawJSON); + public String healthCheck(); +} diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java index cf82e71..38189ac 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java @@ -5,5 +5,6 @@ @Service public interface RssToJsonService { + public JSONObject getJsonifiedXMLUrl(String feed); public JSONObject getJsonFromURL(String url); } diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java index 765de02..b4b423b 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java @@ -2,17 +2,19 @@ import org.json.JSONArray; import org.json.JSONObject; +import org.json.XML; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.context.annotation.Primary; import com.rometools.rome.feed.synd.SyndEntry; import com.rometools.rome.feed.synd.SyndFeed; import edu.wisc.my.rssToJson.dao.RssToJsonDao; - +@Primary @Service public class RsstoJsonServiceImpl implements RssToJsonService { @@ -58,4 +60,10 @@ public JSONObject getJsonFromURL(String endpoint) { stringToClean = stringToClean.replaceAll("\\\\u2014", "-"); return new JSONObject(stringToClean); } + + @Override + public JSONObject getJsonifiedXMLUrl(String feed) { + JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(feed); + return xmlJSONObj; + } } diff --git a/src/main/resources/endpoint.properties b/src/main/resources/endpoint.properties index cc1b10f..1a06a1e 100644 --- a/src/main/resources/endpoint.properties +++ b/src/main/resources/endpoint.properties @@ -12,4 +12,5 @@ uwp-news=https://www.uwp.edu/explore/media/rss.xml working-at-uw=https://working.wisc.edu/feed/ uw-system-payroll-benefits-news=https://uwservice.wisconsin.edu/news/feed/B xkcd=http://xkcd.com/rss.xml +wud=https://union.wisc.edu/events-and-activities/event-calendar/feed/wud.xml sample=https://raw.githubusercontent.com/UW-Madison-DoIT/rssToJson/master/src/main/resources/sampleFile.rss From fac2e2b821edfe07301d272d0365d59538a9c333 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Wed, 21 Nov 2018 12:46:21 -0600 Subject: [PATCH 2/8] fix: correct recursive call --- src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java index 5c8843a..73b6a6a 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java @@ -62,7 +62,7 @@ public JSONObject getXMLFeed(String feedEndpoint) { } private String getEndpointURL(String feed) { - String endpointURL = getEndpointURL(feed); + String endpointURL = env.getProperty(feed); if (endpointURL == null) { logger.warn("No corresponding feed url for requested endpoint {}", feed); return null; From 537f46f45b0526292877a076487199e0285a669c Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 08:42:57 -0600 Subject: [PATCH 3/8] interim commit --- pom.xml | 11 +- .../my/rssToJson/dao/RssToJsonDaoImpl.java | 102 +++++++++++------- src/main/resources/logback.xml | 2 +- 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index 256582a..fa51145 100644 --- a/pom.xml +++ b/pom.xml @@ -66,9 +66,9 @@ json - javax.json - javax.json-api - 1.1.4 + org.apache.httpcomponents + fluent-hc + 4.5.6 org.springframework.boot @@ -96,6 +96,11 @@ rome 1.7.0 + + nio + nio + 1.0.4 + org.apache.httpcomponents httpclient diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java index 73b6a6a..e9fa794 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java @@ -2,15 +2,22 @@ import java.io.InputStream; import java.io.InputStreamReader; - -import org.apache.http.HttpHeaders; +import java.io.ByteArrayInputStream; +import java.io.IOException; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; import org.json.JSONObject; import org.json.XML; -import javax.json.JsonReader; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -35,25 +42,51 @@ public class RssToJsonDaoImpl implements RssToJsonDao{ @Autowired void setEnv(Environment env) { this.env = env; + } + private String httpResponseAsString(String feedName) throws IOException { + logger.error("HTTP Response method " + feedName); + CloseableHttpClient httpclient = HttpClients.createDefault(); + try { + String url = getEndpointURL(feedName); + logger.error("ONE " + url); + HttpGet httpget = new HttpGet(url); + logger.error(httpget.toString()); + + // Create a custom response handler + ResponseHandler responseHandler = new ResponseHandler() { + + @Override + public String handleResponse( + final HttpResponse response) throws ClientProtocolException, IOException { + logger.error("TWO " + response.toString()); + int status = response.getStatusLine().getStatusCode(); + logger.debug(status + " response code "); + if (status >= 200 && status < 300) { + HttpEntity entity = response.getEntity(); + return entity != null ? EntityUtils.toString(entity) : null; + } else { + throw new ClientProtocolException("Unexpected response status: " + status); + } + } + }; + logger.error("ONE POINT FIVE " + httpclient.toString()); + + String responseBody = httpclient.execute(httpget, responseHandler); + logger.error("ONE POINT SIX " + responseBody); + return responseBody; + } finally { + httpclient.close(); + } } + @Override public JSONObject getXMLFeed(String feedEndpoint) { String endpointURL = getEndpointURL(feedEndpoint); JSONObject jsonObject = null; try { - HttpClient client = HttpClientBuilder.create().build(); - HttpGet request = new HttpGet(endpointURL); - request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service"); - HttpResponse response = client.execute(request); - InputStreamReader isr = new InputStreamReader(response.getEntity().getContent()); - StringBuffer stringBuffer = new StringBuffer(); - int i; - while ((i = isr.read()) != -1) { - stringBuffer.append((char) i); - } - String xmlString = stringBuffer.toString(); - jsonObject = XML.toJSONObject(xmlString); + String xmlString = httpResponseAsString(endpointURL); + jsonObject = XML.toJSONObject(xmlString); } catch (Exception e) { logger.warn(e.getMessage()); return null; @@ -62,7 +95,9 @@ public JSONObject getXMLFeed(String feedEndpoint) { } private String getEndpointURL(String feed) { + logger.error("GETTING THE ENDPOINT FOR " + feed); String endpointURL = env.getProperty(feed); + logger.error(endpointURL); if (endpointURL == null) { logger.warn("No corresponding feed url for requested endpoint {}", feed); return null; @@ -73,31 +108,20 @@ private String getEndpointURL(String feed) { @Override @Cacheable(cacheNames="feeds", sync=true) public SyndFeed getRssFeed(String feedEndpoint) { - logger.info("Fetching feed for {} ", feedEndpoint); - //see if property file has corresponding url for requested endpoint - String endpointURL = env.getProperty(feedEndpoint); - if (endpointURL == null){ - logger.warn("No corresponding feed url for requested endpoint {}", - feedEndpoint); - return null; - } - SyndFeed feed = null; - try{ - HttpClient client = HttpClientBuilder.create().build(); - HttpGet request = new HttpGet(endpointURL); - request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service"); - request.setHeader(HttpHeaders.CONTENT_ENCODING, "UTF-8"); - HttpResponse response = client.execute(request); - SyndFeedInput input = new SyndFeedInput(); - feed = input.build(new InputStreamReader(response.getEntity().getContent(), "UTF-8")); - feed.setFeedType("UTF-8"); - logger.debug("CONTENT OF FEED " + endpointURL); - logger.debug(feed.toString()); - - }catch(Exception ex){ - logger.error("Error while fetching xml from {}", endpointURL, ex); - } + try{ + logger.error("INTO THE TRY"); + String result = httpResponseAsString(feedEndpoint); + SyndFeedInput input = new SyndFeedInput(); + InputStream stream = new ByteArrayInputStream(result.getBytes("UTF-8")); + SyndFeed feed = input.build(new InputStreamReader(stream, "UTF-8")); + logger.debug("CONTENT OF FEED " + feedEndpoint); + logger.debug(feed.toString()); return feed; + } catch (Exception e) { + logger.warn("Could not get feed " + feedEndpoint + " " + e.getMessage()); + } + + return null; } } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 1721a41..4fe48eb 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -52,7 +52,7 @@ - + From 6ea8bff1ab5341bccb0cbf9d300b60ab051f0fce Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 11:14:27 -0600 Subject: [PATCH 4/8] fix: refactoring --- .../controller/RssToJsonController.java | 12 ++++++------ .../edu/wisc/my/rssToJson/dao/RssToJsonDao.java | 1 + .../wisc/my/rssToJson/dao/RssToJsonDaoImpl.java | 11 +++-------- .../edu/wisc/my/rssToJson/filter/WudFilter.java | 16 ++++++++-------- .../my/rssToJson/service/RssToJsonService.java | 2 +- .../rssToJson/service/RsstoJsonServiceImpl.java | 9 ++++----- 6 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java index 81ae041..8467bf4 100644 --- a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java +++ b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java @@ -31,13 +31,13 @@ public void setRSSToJSONService(RssToJsonService rssToJsonService) { this.rssToJsonService = rssToJsonService; } - @RequestMapping(value="/rssTransform/{feed}/xml") - public @ResponseBody void getJsonifiedXMLUrl(HttpServletRequest request, HttpServletResponse response, + @RequestMapping(value="/rssTransform/custom/{feed}") + public @ResponseBody void customFeedAsJson(HttpServletRequest request, HttpServletResponse response, @PathVariable String feed) { - - JSONObject jsonFromFeed = rssToJsonService.getJsonifiedXMLUrl(feed); + + JSONObject jsonFromFeed = rssToJsonService.getCustomizedUrl(feed); if (jsonFromFeed == null) { - logger.error("No feed for endpoint {}", feed); + logger.warn("No feed for endpoint {}", feed); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } else { /* The filter is a custom class based on the name of your endpoint. @@ -63,7 +63,7 @@ public void setRSSToJSONService(RssToJsonService rssToJsonService) { } @RequestMapping(value="/rssTransform/{feed}") - public @ResponseBody void getJsonifiedRssUrl(HttpServletRequest request, + public @ResponseBody void rssAsJson(HttpServletRequest request, HttpServletResponse response, @PathVariable String feed) { logger.debug("Attempting to retrieve feed for endpoint {}", feed); diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java index 5418524..77e3a0d 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java @@ -6,5 +6,6 @@ public interface RssToJsonDao{ public JSONObject getXMLFeed(String feedEndpoint); public SyndFeed getRssFeed(String feedEndpoint); + public String getEndpointURL(String feedEndpoint); } diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java index e9fa794..80d2f34 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java @@ -43,12 +43,10 @@ public class RssToJsonDaoImpl implements RssToJsonDao{ void setEnv(Environment env) { this.env = env; } - private String httpResponseAsString(String feedName) throws IOException { - logger.error("HTTP Response method " + feedName); + private String httpResponseAsString(String url) throws IOException { + logger.error("HTTP Response method " + url); CloseableHttpClient httpclient = HttpClients.createDefault(); try { - String url = getEndpointURL(feedName); - logger.error("ONE " + url); HttpGet httpget = new HttpGet(url); logger.error(httpget.toString()); @@ -69,8 +67,6 @@ public String handleResponse( } } }; - logger.error("ONE POINT FIVE " + httpclient.toString()); - String responseBody = httpclient.execute(httpget, responseHandler); logger.error("ONE POINT SIX " + responseBody); return responseBody; @@ -94,7 +90,7 @@ public JSONObject getXMLFeed(String feedEndpoint) { return jsonObject; } - private String getEndpointURL(String feed) { + public String getEndpointURL(String feed) { logger.error("GETTING THE ENDPOINT FOR " + feed); String endpointURL = env.getProperty(feed); logger.error(endpointURL); @@ -109,7 +105,6 @@ private String getEndpointURL(String feed) { @Cacheable(cacheNames="feeds", sync=true) public SyndFeed getRssFeed(String feedEndpoint) { try{ - logger.error("INTO THE TRY"); String result = httpResponseAsString(feedEndpoint); SyndFeedInput input = new SyndFeedInput(); InputStream stream = new ByteArrayInputStream(result.getBytes("UTF-8")); diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java index a79a808..75fa211 100644 --- a/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java +++ b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java @@ -14,8 +14,9 @@ public WudFilter(){ } public JSONObject getFilteredJSON(JSONObject rawJSON){ ObjectMapper om = new ObjectMapper(); - JSONObject responseObj = new JSONObject(); JSONObject feedInfo = new JSONObject(); + JSONObject feed = new JSONObject(); + JSONArray items = new JSONArray(); try{ JsonNode rootNode = om.readTree(rawJSON.toString()); feedInfo.put("title", rootNode.findValue("title").asText()); @@ -27,7 +28,6 @@ public JSONObject getFilteredJSON(JSONObject rawJSON){ JsonNode events = rootNode.findValue("event"); Iterator iter = events.elements(); - ArrayList eventNodes = new ArrayList(); while(iter.hasNext()){ JSONObject item = new JSONObject(); @@ -35,17 +35,17 @@ public JSONObject getFilteredJSON(JSONObject rawJSON){ item.put("title", anEvent.findValue("event_title").asText()); item.put("link", anEvent.findValue("url").asText()); item.put("description",anEvent.findValue("short_description").asText()); - JSONObject thisItem = new JSONObject(); - thisItem.put("item", item); - eventNodes.add(thisItem); + items.put(item); } - JSONArray allEvents = new JSONArray(eventNodes); - feedInfo.put("items",allEvents); + feed.put("feed", feedInfo); + feed.put("items",items); + }catch(Exception e){ logger.error(e.getMessage()); }; - return feedInfo; + feed.put("status", "ok"); + return feed; } public String healthCheck(){ return "WudFilter health check"; diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java index 38189ac..81f72f0 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java @@ -5,6 +5,6 @@ @Service public interface RssToJsonService { - public JSONObject getJsonifiedXMLUrl(String feed); + public JSONObject getCustomizedUrl(String feed); public JSONObject getJsonFromURL(String url); } diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java index b4b423b..63c1cef 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java @@ -2,11 +2,9 @@ import org.json.JSONArray; import org.json.JSONObject; -import org.json.XML; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.context.annotation.Primary; @@ -29,7 +27,8 @@ void setRssToJsonDao(RssToJsonDao rssToJsonDao){ @Override public JSONObject getJsonFromURL(String endpoint) { - SyndFeed feed = rssToJsonDao.getRssFeed(endpoint); + String url = rssToJsonDao.getEndpointURL(endpoint); + SyndFeed feed = rssToJsonDao.getRssFeed(url); if(feed == null){ logger.warn("No feed returned for endpoint: {}", endpoint); return null; @@ -62,8 +61,8 @@ public JSONObject getJsonFromURL(String endpoint) { } @Override - public JSONObject getJsonifiedXMLUrl(String feed) { - JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(feed); + public JSONObject getCustomizedUrl(String endpoint) { + JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(endpoint); return xmlJSONObj; } } From cd734e49436afb8c4c659fc02db6c8d51c94d627 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 11:16:19 -0600 Subject: [PATCH 5/8] fix: refactoring --- .../edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java index 63c1cef..c5a3df8 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java @@ -12,8 +12,8 @@ import com.rometools.rome.feed.synd.SyndFeed; import edu.wisc.my.rssToJson.dao.RssToJsonDao; -@Primary +@Primary @Service public class RsstoJsonServiceImpl implements RssToJsonService { protected final Logger logger = LoggerFactory.getLogger(getClass()); From 6fa06ee50c664c3e27698171c940890e6a287f04 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 11:23:16 -0600 Subject: [PATCH 6/8] fix: correct README --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 741277c..8ce6fb9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -82,7 +82,7 @@ returns something like ] } ``` -##Custom Filters /{webapp-name}/rssTransform/prop/{key}/xml +##Custom Filters /{webapp-name}/rssTransform/prop/custom/{key} Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide. From c4725805eb17369d1ec9ae1ca0483a613a478e16 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 11:26:42 -0600 Subject: [PATCH 7/8] fix: remove unused dependency --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index fa51145..a7c7fee 100644 --- a/pom.xml +++ b/pom.xml @@ -96,11 +96,6 @@ rome 1.7.0 - - nio - nio - 1.0.4 - org.apache.httpcomponents httpclient From 4f4e8ec65611f8a03e233d188ed39892d5b97130 Mon Sep 17 00:00:00 2001 From: Doug Reed Date: Mon, 26 Nov 2018 11:27:13 -0600 Subject: [PATCH 8/8] Update README.md --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 8ce6fb9..e1ac900 100644 --- a/docs/README.md +++ b/docs/README.md @@ -82,7 +82,7 @@ returns something like ] } ``` -##Custom Filters /{webapp-name}/rssTransform/prop/custom/{key} +##Custom Filters /{webapp-name}/rssTransform/custom/{key} Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide.