Add multiple sources for feeds

Dockerfile

@@ -0,0 +1,32 @@
+FROM node
+
+RUN adduser user
+USER user
+
+
+USER user
+WORKDIR /home/user
+COPY . .
+RUN yarn install --modules-folder=/home/user/node_modules
+COPY /scripts/main_loop.sh .
+COPY /scripts/harden.sh .
+
+USER root
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt-get install --yes nginx
+
+ENV TZ=Europe/Vienna
+ENV LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+
+COPY nginx.conf /etc/nginx/nginx.conf
+RUN mkdir -p /var/lib/nginx
+RUN chown -R user:user /var/lib/nginx
+RUN chown -R user:user /var/log/nginx
+RUN chown -R user:user .
+
+RUN touch /run/nginx.pid && chown -R user:user /run/nginx.pid
+
+RUN ./harden.sh user
+USER user
+
+CMD /home/user/main_loop.sh

bin/generate-html.mjs

@@ -6,50 +6,97 @@ import fs from 'fs-extra'
 import psl from 'psl'
 import html2text from 'html2plaintext'
 
+function getArgs () {
+    const args = {};
+    process.argv
+        .slice(2, process.argv.length)
+        .forEach( arg => {
+        // long arg
+        if (arg.slice(0,2) === '--') {
+            const longArg = arg.split('=');
+            const longArgFlag = longArg[0].slice(2,longArg[0].length);
+            const longArgValue = longArg.length > 1 ? longArg[1] : true;
+            args[longArgFlag] = longArgValue;
+        }
+        // flags
+        else if (arg[0] === '-') {
+            const flags = arg.slice(1,arg.length).split('');
+            flags.forEach(flag => {
+            args[flag] = true;
+            });
+        }
+    });
+    return args;
+}
+
 ;(async () => {
-	await fs.ensureDir('cache/hn/item')
-	await fs.ensureDir('cache/url')
-
-	await init_titles()
-
+
+	const args = getArgs();
+
 	const stories = []
-	for (const storyid of (await hnget('topstories')).slice(0, 30)) {
-		const story = await getitem(storyid)
-		if (story.type !== 'story') continue // Filter out jobs
-		story.keyword = title_keyword(story.title)
-		if (story.text) { // Self post
-			story.url = 'https://news.ycombinator.com/item?id='+story.id
-			story.domain = 'news.ycombinator.com'
-			story.paragraph = html2text(story.text.split('<p>')[0])
-			story.image = false
-		} else {
-			story.domain = psl.parse(new URL(story.url).hostname).domain
-			const info = await page_info(story.url)
-			story.paragraph = info.paragraph
-			story.image = info.image
-		}
-		stories.push(story)
-	}
-
 	const jobs = []
-	for (const jobid of (await hnget('jobstories')).slice(0, 3)) {
-		const job = await getitem(jobid)
-		job.domain = psl.parse(new URL(job.url).hostname).domain
-		const split = job.title.split(' ')
-		const splitix = (split.findIndex(w => w.toLowerCase() === 'hiring') || 3)+1
-		job.title1 = split.slice(0, splitix).join(' ')
-		job.title2 = split.slice(splitix).join(' ')
-		jobs.push(job)
+	const source_urls=['https://hacker-news.firebaseio.com/v0/']
+	const story_urls=['https://news.ycombinator.com/item?id=']
+	const story_domains=['news.ycombinator.com']
+	const cache_paths=['hacker-news']
+
+
+	for (var i=0; i<source_urls.length; i++){
+		var source_url=source_urls[i]
+		var story_url=story_urls[i]
+		var story_domain=story_domains[i]
+		var cache_path=args[cache_paths[i]] // i.e. pass --hacker-news=hn (dirty hack but I'm not a JS dev.)
+
+		await fs.ensureDir(`cache/${cache_path}/item`)
+		await fs.ensureDir('cache/url')
+		await init_titles(cache_path)
 	}
-
+
+	for (var i=0; i<source_urls.length; i++){
+		var source_url=source_urls[i]
+		var story_url=story_urls[i]
+		var story_domain=story_domains[i]
+		var cache_path=args[cache_paths[i]]
+
+		console.log("Cache path: " + cache_path);
+
+		var storyids=await hnget(source_url, 'topstories',cache_path)
+
+		for (var storyid of (storyids).slice(0, 30)) {
+			var story = await getitem(source_url, storyid, cache_path)
+			if (story.type !== 'story') continue // Filter out jobs
+			story.keyword = title_keyword(story.title)
+			if (story.text) { // Self post
+				story.url = story_url+story.id
+				story.domain = story_domain
+				story.paragraph = html2text(story.text.split('<p>')[0])
+				story.image = false
+			} else {
+				story.domain = psl.parse(new URL(story.url).hostname).domain
+				const info = await page_info(story.url)
+				story.paragraph = info.paragraph
+				story.image = info.image
+			}
+			stories.push(story)
+		}
+
+		const jobstories = await hnget(source_url, 'jobstories', cache_path)
+		if(jobstories != null)
+		{
+			for (const jobid of jobstories.slice(0, 3)) {
+				const job = await getitem(source_url, jobid, cache_path)
+				const split = job.title.split(' ')
+				const splitix = (split.findIndex(w => w.toLowerCase() === 'hiring') || 3)+1
+				job.title1 = split.slice(0, splitix).join(' ')
+				job.title2 = split.slice(splitix).join(' ')
+				jobs.push(job)
+			}
+		}
+	}
+
 	await fs.writeFile('index.html', pug.renderFile('index.pug', {
 		stories,
 		jobs,
-		date: new Date(1000*Math.max(...stories.map(s => s.time))).toLocaleString('en-US', {
-			timeZone: 'UTC',
-			dateStyle: 'full',
-			timeStyle: 'short',
-			timeZoneName: 'short',
-		}),
+		date: new Intl.DateTimeFormat('en', { dateStyle: "medium", timeStyle: "medium" }).format(new Date()),
 	}))
 })().then(null, err => {throw err})

lib/hn.mjs

@@ -1,20 +1,23 @@
 import fetch from 'node-fetch'
 import fs from 'fs-extra'
 
-export async function hnget(q) {
-	const cache = `cache/hn/${q}.json`
+export async function hnget(source, q, cache_path) {
+	const cache = `cache/${cache_path}/${q}.json`
 	if (await fs.exists(cache)) {
 		return fs.readJson(cache)
 	}
-	const res = await fetch(`https://hacker-news.firebaseio.com/v0/${q}.json`)
-	if (!res.ok) {
-		throw new Error(`HTTP error: ${res.status} for ${q}`)
+	const url = `${source}${q}.json`
+
+	const res = await fetch(url)
+	if (!res.ok)
+	{
+		throw new Error(`HTTP error: ${res.status} for ${source}${q}.json`)
 	}
 	const ret = await res.json()
 	await fs.writeJson(cache, ret)
 	return ret
 }
 
-export async function getitem(itemid) {
-	return await hnget(`item/${itemid}`)
+export async function getitem(source,itemid, cache_path) {
+	return await hnget(source,`item/${itemid}`, cache_path)
 }

lib/keywords.mjs

@@ -32,9 +32,9 @@ async function titles_file() {
 }
 
 let tokcount
-export async function init_titles() {
+export async function init_titles(cache_path) {
 	const titles = await titles_file()
-	for (const storyfile of await fs.readdir('cache/hn/item/')) {
+	for (const storyfile of await fs.readdir(`cache/${cache_path}/item/`)) {
 		const storyid = storyfile.split('.')[0]
 		const story = await getitem(storyid)
 		if (story.type !== 'story') {
@@ -43,7 +43,7 @@ export async function init_titles() {
 		titles[story.id] = story.title
 	}
 	await fs.writeJson('titles.json', titles)
-	
+
 	tokcount = {}
 	for (const storyid in titles) {
 		const title = titles[storyid]

lib/page-extract.mjs

@@ -1,7 +1,7 @@
 import crypto from 'crypto'
 import fetch from 'node-fetch'
 import fs from 'fs-extra'
-import cheerio from 'cheerio'
+import * as cheerio from "cheerio"
 import compare from 'compare-function'
 import URLToolkit from 'url-toolkit'
 import htmlEntities from 'html-entities'
@@ -20,7 +20,8 @@ async function gethtml(url) {
 	}
 	const res = await fetch(url)
 	if (!res.ok) {
-		throw new Error(`HTTP error: ${res.status} for ${url}`)
+		console.log(`HTTP error. Skipping: ${res.status} for ${url}`)
+		return ''
 	}
 	let ret
 	if (res.headers.get('Content-Type').startsWith('text/html')) {
@@ -132,7 +133,7 @@ export async function page_info(url) {
 		normalizeWhitespace: true,
 	}})
 	$('script').remove()
-	
+
 	return {
 		paragraph: getParagraph($),
 		image: getImage(url, $),

nginx.conf

@@ -0,0 +1,40 @@
+events {
+
+}
+
+http {
+    include /etc/nginx/conf.d/*.conf;
+    server {
+        listen       8080 default_server;
+        listen       [::]:8080 default_server;
+        server_name  _;
+        #root         /usr/share/nginx/html;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        location / {
+            root /home/user;
+            autoindex on;
+        }
+        location = /health {
+            access_log off;
+            add_header 'Content-Type' 'application/json';
+            return 200 '{"status":"OK"}';
+        }
+        location = /ready {
+            access_log off;
+            add_header 'Content-Type' 'application/json';
+            return 200 '{"status":"OK"}';
+        }
+
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+}

package.json

@@ -5,6 +5,7 @@
     "fs-extra": "^9.0.1",
     "html-entities": "^1.3.1",
     "html2plaintext": "^2.1.2",
+    "punycode": "^2.3.1",
     "node-fetch": "^2.6.0",
     "nodemon": "^2.0.4",
     "psl": "^1.8.0",