Crawler Fetcher

.docker/start.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 
-# source /root/.nvm/nvm.sh
-
-mix phoenix.server
+mix deps.compile \
+  && mix compile \
+  && mix phoenix.digest \
+  && mix ecto.create \
+  && mix phoenix.server
 
 echo "Press [CTRL+C] to stop.."
 while :
 do
-	sleep 10
+	sleep 60
 done

Dockerfile

@@ -1,11 +1,9 @@
-# FROM bitwalker/alpine-erlang:19.2.1
 FROM bitwalker/alpine-elixir-phoenix:latest
 
 # Set the locale
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
-ENV MIX_ENV prod
 ENV TERM xterm
 
 RUN apk add --update \
@@ -16,18 +14,6 @@ RUN apk add --update \
   make \
   gcc \
   g++ \
-#  erlang \
-#  erlang-mnesia \
-#  erlang-public-key \
-#  erlang-crypto \
-#  erlang-ssl \
-#  erlang-sasl \
-#  erlang-asn1 \
-#  erlang-inets \
-#  erlang-os-mon \
-#  erlang-xmerl \
-#  erlang-eldap \
-#  erlang-syntax-tools \
   && rm -rf /var/cache/apk/*
 
 # Switch to directory with sources
@@ -41,13 +27,7 @@ RUN mix local.hex --force && \
 ADD . .
 
 RUN mix deps.get \
-  mix deps.compile
-
-RUN npm install
-
-RUN mix compile \
-  && mix phoenix.digest \
-  && mix ecto.create
+    && npm install
 
 ADD .docker/start.sh /start.sh
 

client/src/containers/Menu/Menu.js

@@ -0,0 +1,119 @@
+import React, { Component, PropTypes } from 'react';
+
+import { connect } from 'react-redux';
+import { push } from 'react-router-redux';
+
+import { IndexLink } from 'react-router';
+import { LinkContainer } from 'react-router-bootstrap';
+
+import Nav from 'react-bootstrap/lib/Nav';
+import Navbar from 'react-bootstrap/lib/Navbar';
+import NavItem from 'react-bootstrap/lib/NavItem';
+
+import config from '../../config';
+
+import * as authActions from '../../redux/modules/auth';
+import * as socketActions from '../../redux/modules/socket';
+import * as workerActions from '../../redux/modules/workers';
+
+@connect(
+  state => ({
+    user: state.auth.user
+  }),
+  {
+    pushState: push,
+    ...authActions,
+    ...socketActions,
+    ...workerActions
+  }
+)
+export default class Menu extends Component {
+  static propTypes = {
+    pushState: PropTypes.func.isRequired,
+    signOut: PropTypes.func,
+
+    user: PropTypes.object,
+  };
+
+  render() {
+    const styles = require('./Menu.scss');
+
+    return (
+      <Navbar fixedTop>
+        <Navbar.Header>
+          <Navbar.Brand>
+            <IndexLink to="/" activeStyle={{ color: '#33e0ff' }}>
+              <span className={styles.brand} />
+              <span>{config.app.name}</span>
+            </IndexLink>
+          </Navbar.Brand>
+          <Navbar.Toggle/>
+        </Navbar.Header>
+
+        <Navbar.Collapse>
+          <Nav navbar>
+            {this.props.user &&
+            <LinkContainer to="clients">
+              <NavItem>Clients</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <LinkContainer to="crawlers">
+              <NavItem>Crawlers</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <LinkContainer to="workers">
+              <NavItem>Workers</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <LinkContainer to="progress">
+              <NavItem>Progress</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <LinkContainer to="admin">
+              <NavItem>Admin</NavItem>
+            </LinkContainer>
+            }
+          </Nav>
+
+          <Nav navbar pullRight>
+            {!this.props.user &&
+            <LinkContainer to="signin">
+              <NavItem>Sign In</NavItem>
+            </LinkContainer>
+            }
+
+            {!this.props.user &&
+            <LinkContainer to="signup">
+              <NavItem>Sign Up</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <LinkContainer to="profile">
+              <NavItem>{this.props.user.email}</NavItem>
+            </LinkContainer>
+            }
+
+            {this.props.user &&
+            <NavItem
+              onClick={ () => {
+                this.props.signOut().then(() => this.props.pushState('/'));
+              }}
+            >
+              Sign Out
+            </NavItem>
+            }
+          </Nav>
+        </Navbar.Collapse>
+      </Navbar>
+    );
+  }
+}

client/src/containers/Menu/Menu.scss

@@ -0,0 +1,22 @@
+.app {
+  .brand {
+    position: absolute;
+    $size: 40px;
+    top: 5px;
+    left: 5px;
+    display: inline-block;
+    background: url("/images/spider.png") no-repeat center center;
+    width: $size;
+    height: $size;
+    background-size: 80%;
+    margin: 0 10px 0 0;
+    border-radius: $size / 2;
+  }
+  nav :global(.fa) {
+    font-size: 2em;
+    line-height: 20px;
+  }
+  .appContent {
+    margin: 50px 0; // for fixed navbar
+  }
+}

client/src/containers/Menu/index.js

@@ -0,0 +1,3 @@
+import Menu from './Menu';
+
+export default Menu;

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       - "9200:9200"
       - "9300:9300"
   couchbase:
-    image: korczis/couchbase:4.5.1
+    image: couchbase/server:enterprise-4.6.0
     ports:
       - "8091:8091"
       - "8092:8092"
@@ -19,6 +19,8 @@ services:
       - "18091:18091"
       - "18092:18092"
       - "18093:18093"
+    volumes:
+      - "/data/couchbase:/opt/couchbase/var"
     depends_on:
       - elasticsearch
     ulimits:

lib/microcrawler_webapp.ex

@@ -24,6 +24,8 @@ defmodule MicrocrawlerWebapp do
       worker(MicrocrawlerWebapp.ActiveWorkers, []),
     ]
 
+    MicrocrawlerWebapp.CrawlerManager.update_crawlers()
+
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: MicrocrawlerWebapp.Supervisor]

lib/microcrawler_webapp/commander.ex

@@ -0,0 +1,9 @@
+defmodule MicrocrawlerWebapp.Commander do
+  @moduledoc """
+  Command Line Wrapper
+  """
+
+  def cmd(cmd, args) do
+    System.cmd(cmd, args)
+  end
+end

lib/microcrawler_webapp/crawler_manager.ex

@@ -0,0 +1,77 @@
+defmodule MicrocrawlerWebapp.CrawlerManager do
+  @moduledoc """
+  Module for managing crawlers
+  """
+
+  @default_prefix "microcrawler-crawler-"
+
+  def install_crawler(crawler) do
+    {:ok, name} = Map.fetch(crawler, "name")
+    {:ok, version} = Map.fetch(crawler, "version")
+
+    package_full_name = "#{name}@#{version}"
+    {output, _code} = MicrocrawlerWebapp.Commander.cmd("npm", ["install", package_full_name])
+
+    {name, version}
+  end
+
+  def install_crawlers(crawlers) do
+    Enum.map(crawlers, fn(crawler) -> install_crawler(crawler) end)
+  end
+
+  def query_crawlers(input_query \\ @default_prefix) do
+    MicrocrawlerWebapp.Npm.query(input_query)
+    |> Enum.filter(fn(x) -> String.starts_with?(Map.fetch!(x, "name"), input_query) end)
+    |> Enum.reduce(%{}, fn(e, a) ->
+      Map.put(a, Map.fetch!(e, "name"), e)
+    end)
+  end
+
+  def update_crawlers(input_query \\ @default_prefix) do
+    local_crawlers = list_existing(input_query)
+    remote_crawlers = query_crawlers(input_query)
+
+    to_install = Enum.map(Map.keys(remote_crawlers), fn(k) ->
+      Map.fetch!(remote_crawlers, k)
+    end)
+    |> Enum.filter(fn(x) ->
+      case Map.fetch(local_crawlers, Map.fetch!(x, "name")) do
+        :error -> true
+        {:ok, item} ->
+          local_version = Map.fetch!(item, :version)
+          remote_version = Map.fetch!(x, "version")
+          local_version != remote_version
+      end
+    end)
+
+    install_crawlers(to_install)
+
+    path = "data/crawlers.json"
+    write_crawlers_to_file(path, remote_crawlers)
+
+    remote_crawlers
+  end
+
+  def write_crawlers_to_file(path, crawlers) do
+    {:ok, file} = File.open(path, [:write])
+    nice_json = Poison.encode_to_iodata!(crawlers, pretty: true)
+    IO.binwrite(file, nice_json)
+    File.close(file)
+  end
+
+  def list_existing(input_query \\ @default_prefix) do
+    File.ls!("node_modules")
+    |> Enum.filter(fn(x) -> String.starts_with?(x, input_query) end)
+    |> Enum.map(fn(x) -> File.read!("node_modules/#{x}/package.json") end)
+    |> Enum.map(fn(x) -> Poison.decode!(x) end)
+    |> Enum.map(fn(x) -> %{
+      name: Map.fetch!(x, "name"),
+      version: Map.fetch!(x, "version"),
+      description: Map.fetch!(x, "description"),
+    } end)
+    |> Enum.reduce(%{}, fn(e, a) ->
+      Map.put(a, Map.fetch!(e, :name), e)
+    end)
+  end
+
+end

lib/microcrawler_webapp/npm.ex

@@ -0,0 +1,24 @@
+defmodule MicrocrawlerWebapp.Npm do
+  @moduledoc """
+  Module for querying https://npmjs.org
+  """
+
+  def query(input_query) do
+    args = [
+      "search",
+      "--json",
+      input_query # TODO: Sanitize input_query!
+    ]
+
+    {output, _code} = MicrocrawlerWebapp.Commander.cmd("npm", args)
+
+    case Poison.decode(output) do
+      {:ok, res} ->
+        res
+      _ -> nil
+    end
+  end
+
+  def update(package, version) do
+  end
+end

mix.exs

@@ -3,7 +3,7 @@ defmodule MicrocrawlerWebapp.Mixfile do
 
   def project do
     [app: :microcrawler_webapp,
-     version: "0.0.7",
+     version: "0.0.9",
      elixir: "~> 1.2",
      elixirc_paths: elixirc_paths(Mix.env),
      compilers: [:phoenix, :gettext] ++ Mix.compilers,
@@ -44,12 +44,13 @@ defmodule MicrocrawlerWebapp.Mixfile do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
-      {:comeonin, "~> 2.6"},
+      {:comeonin, "~> 3.0"},
       {:credo, "~> 0.7", only: [:dev, :test]},
       {:dialyxir, "~> 0.5", only: [:dev]},
       {:elastix, "~> 0.3"},
       {:guardian, "~> 0.14"},
       {:httpoison, "~> 0.11"},
+      {:passport, git: "https://github.com/opendrops/passport.git"},
       {:phoenix, "~> 1.2.3", override: true},
       {:phoenix_pubsub, "~> 1.0"},
       {:phoenix_ecto, "~> 3.2"},