diff --git a/.docker/start.sh b/.docker/start.sh index d3e6d06..798c1fe 100755 --- a/.docker/start.sh +++ b/.docker/start.sh @@ -1,11 +1,13 @@ #!/bin/bash -# source /root/.nvm/nvm.sh - -mix phoenix.server +mix deps.compile \ + && mix compile \ + && mix phoenix.digest \ + && mix ecto.create \ + && mix phoenix.server echo "Press [CTRL+C] to stop.." while : do - sleep 10 + sleep 60 done diff --git a/Dockerfile b/Dockerfile index d7f8f98..cdf3b97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,9 @@ -# FROM bitwalker/alpine-erlang:19.2.1 FROM bitwalker/alpine-elixir-phoenix:latest # Set the locale ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV MIX_ENV prod ENV TERM xterm RUN apk add --update \ @@ -16,18 +14,6 @@ RUN apk add --update \ make \ gcc \ g++ \ -# erlang \ -# erlang-mnesia \ -# erlang-public-key \ -# erlang-crypto \ -# erlang-ssl \ -# erlang-sasl \ -# erlang-asn1 \ -# erlang-inets \ -# erlang-os-mon \ -# erlang-xmerl \ -# erlang-eldap \ -# erlang-syntax-tools \ && rm -rf /var/cache/apk/* # Switch to directory with sources @@ -41,13 +27,7 @@ RUN mix local.hex --force && \ ADD . . RUN mix deps.get \ - mix deps.compile - -RUN npm install - -RUN mix compile \ - && mix phoenix.digest \ - && mix ecto.create + && npm install ADD .docker/start.sh /start.sh diff --git a/client/src/containers/Menu/Menu.js b/client/src/containers/Menu/Menu.js new file mode 100644 index 0000000..26929aa --- /dev/null +++ b/client/src/containers/Menu/Menu.js @@ -0,0 +1,119 @@ +import React, { Component, PropTypes } from 'react'; + +import { connect } from 'react-redux'; +import { push } from 'react-router-redux'; + +import { IndexLink } from 'react-router'; +import { LinkContainer } from 'react-router-bootstrap'; + +import Nav from 'react-bootstrap/lib/Nav'; +import Navbar from 'react-bootstrap/lib/Navbar'; +import NavItem from 'react-bootstrap/lib/NavItem'; + +import config from '../../config'; + +import * as authActions from '../../redux/modules/auth'; +import * as socketActions from '../../redux/modules/socket'; +import * as workerActions from '../../redux/modules/workers'; + +@connect( + state => ({ + user: state.auth.user + }), + { + pushState: push, + ...authActions, + ...socketActions, + ...workerActions + } +) +export default class Menu extends Component { + static propTypes = { + pushState: PropTypes.func.isRequired, + signOut: PropTypes.func, + + user: PropTypes.object, + }; + + render() { + const styles = require('./Menu.scss'); + + return ( + + + + + + {config.app.name} + + + + + + + + + + + + ); + } +} diff --git a/client/src/containers/Menu/Menu.scss b/client/src/containers/Menu/Menu.scss new file mode 100644 index 0000000..31283dc --- /dev/null +++ b/client/src/containers/Menu/Menu.scss @@ -0,0 +1,22 @@ +.app { + .brand { + position: absolute; + $size: 40px; + top: 5px; + left: 5px; + display: inline-block; + background: url("/images/spider.png") no-repeat center center; + width: $size; + height: $size; + background-size: 80%; + margin: 0 10px 0 0; + border-radius: $size / 2; + } + nav :global(.fa) { + font-size: 2em; + line-height: 20px; + } + .appContent { + margin: 50px 0; // for fixed navbar + } +} diff --git a/client/src/containers/Menu/index.js b/client/src/containers/Menu/index.js new file mode 100644 index 0000000..0d5d975 --- /dev/null +++ b/client/src/containers/Menu/index.js @@ -0,0 +1,3 @@ +import Menu from './Menu'; + +export default Menu; diff --git a/docker-compose.yml b/docker-compose.yml index 5032158..7d694a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - "9200:9200" - "9300:9300" couchbase: - image: korczis/couchbase:4.5.1 + image: couchbase/server:enterprise-4.6.0 ports: - "8091:8091" - "8092:8092" @@ -19,6 +19,8 @@ services: - "18091:18091" - "18092:18092" - "18093:18093" + volumes: + - "/data/couchbase:/opt/couchbase/var" depends_on: - elasticsearch ulimits: diff --git a/lib/microcrawler_webapp.ex b/lib/microcrawler_webapp.ex index 5ab074f..5674684 100644 --- a/lib/microcrawler_webapp.ex +++ b/lib/microcrawler_webapp.ex @@ -24,6 +24,8 @@ defmodule MicrocrawlerWebapp do worker(MicrocrawlerWebapp.ActiveWorkers, []), ] + MicrocrawlerWebapp.CrawlerManager.update_crawlers() + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: MicrocrawlerWebapp.Supervisor] diff --git a/lib/microcrawler_webapp/commander.ex b/lib/microcrawler_webapp/commander.ex new file mode 100644 index 0000000..bd8134d --- /dev/null +++ b/lib/microcrawler_webapp/commander.ex @@ -0,0 +1,9 @@ +defmodule MicrocrawlerWebapp.Commander do + @moduledoc """ + Command Line Wrapper + """ + + def cmd(cmd, args) do + System.cmd(cmd, args) + end +end diff --git a/lib/microcrawler_webapp/crawler_manager.ex b/lib/microcrawler_webapp/crawler_manager.ex new file mode 100644 index 0000000..7240b09 --- /dev/null +++ b/lib/microcrawler_webapp/crawler_manager.ex @@ -0,0 +1,77 @@ +defmodule MicrocrawlerWebapp.CrawlerManager do + @moduledoc """ + Module for managing crawlers + """ + + @default_prefix "microcrawler-crawler-" + + def install_crawler(crawler) do + {:ok, name} = Map.fetch(crawler, "name") + {:ok, version} = Map.fetch(crawler, "version") + + package_full_name = "#{name}@#{version}" + {output, _code} = MicrocrawlerWebapp.Commander.cmd("npm", ["install", package_full_name]) + + {name, version} + end + + def install_crawlers(crawlers) do + Enum.map(crawlers, fn(crawler) -> install_crawler(crawler) end) + end + + def query_crawlers(input_query \\ @default_prefix) do + MicrocrawlerWebapp.Npm.query(input_query) + |> Enum.filter(fn(x) -> String.starts_with?(Map.fetch!(x, "name"), input_query) end) + |> Enum.reduce(%{}, fn(e, a) -> + Map.put(a, Map.fetch!(e, "name"), e) + end) + end + + def update_crawlers(input_query \\ @default_prefix) do + local_crawlers = list_existing(input_query) + remote_crawlers = query_crawlers(input_query) + + to_install = Enum.map(Map.keys(remote_crawlers), fn(k) -> + Map.fetch!(remote_crawlers, k) + end) + |> Enum.filter(fn(x) -> + case Map.fetch(local_crawlers, Map.fetch!(x, "name")) do + :error -> true + {:ok, item} -> + local_version = Map.fetch!(item, :version) + remote_version = Map.fetch!(x, "version") + local_version != remote_version + end + end) + + install_crawlers(to_install) + + path = "data/crawlers.json" + write_crawlers_to_file(path, remote_crawlers) + + remote_crawlers + end + + def write_crawlers_to_file(path, crawlers) do + {:ok, file} = File.open(path, [:write]) + nice_json = Poison.encode_to_iodata!(crawlers, pretty: true) + IO.binwrite(file, nice_json) + File.close(file) + end + + def list_existing(input_query \\ @default_prefix) do + File.ls!("node_modules") + |> Enum.filter(fn(x) -> String.starts_with?(x, input_query) end) + |> Enum.map(fn(x) -> File.read!("node_modules/#{x}/package.json") end) + |> Enum.map(fn(x) -> Poison.decode!(x) end) + |> Enum.map(fn(x) -> %{ + name: Map.fetch!(x, "name"), + version: Map.fetch!(x, "version"), + description: Map.fetch!(x, "description"), + } end) + |> Enum.reduce(%{}, fn(e, a) -> + Map.put(a, Map.fetch!(e, :name), e) + end) + end + +end diff --git a/lib/microcrawler_webapp/npm.ex b/lib/microcrawler_webapp/npm.ex new file mode 100644 index 0000000..ff28676 --- /dev/null +++ b/lib/microcrawler_webapp/npm.ex @@ -0,0 +1,24 @@ +defmodule MicrocrawlerWebapp.Npm do + @moduledoc """ + Module for querying https://npmjs.org + """ + + def query(input_query) do + args = [ + "search", + "--json", + input_query # TODO: Sanitize input_query! + ] + + {output, _code} = MicrocrawlerWebapp.Commander.cmd("npm", args) + + case Poison.decode(output) do + {:ok, res} -> + res + _ -> nil + end + end + + def update(package, version) do + end +end diff --git a/mix.exs b/mix.exs index bbe3b58..2f82c4c 100644 --- a/mix.exs +++ b/mix.exs @@ -3,7 +3,7 @@ defmodule MicrocrawlerWebapp.Mixfile do def project do [app: :microcrawler_webapp, - version: "0.0.7", + version: "0.0.9", elixir: "~> 1.2", elixirc_paths: elixirc_paths(Mix.env), compilers: [:phoenix, :gettext] ++ Mix.compilers, @@ -44,12 +44,13 @@ defmodule MicrocrawlerWebapp.Mixfile do # Type `mix help deps` for examples and options. defp deps do [ - {:comeonin, "~> 2.6"}, + {:comeonin, "~> 3.0"}, {:credo, "~> 0.7", only: [:dev, :test]}, {:dialyxir, "~> 0.5", only: [:dev]}, {:elastix, "~> 0.3"}, {:guardian, "~> 0.14"}, {:httpoison, "~> 0.11"}, + {:passport, git: "https://github.com/opendrops/passport.git"}, {:phoenix, "~> 1.2.3", override: true}, {:phoenix_pubsub, "~> 1.0"}, {:phoenix_ecto, "~> 3.2"}, diff --git a/mix.lock b/mix.lock index 1afe742..08cd793 100644 --- a/mix.lock +++ b/mix.lock @@ -1,26 +1,28 @@ -%{"amqp": {:hex, :amqp, "0.1.4", "5d9112c3a850234559073c23fda173364dfd11d8ce8d0138cd46f653fdc55f08", [:mix], [{:amqp_client, ">= 3.5.6", [hex: :amqp_client, optional: false]}]}, - "amqp_client": {:hex, :amqp_client, "3.6.8", "0388e50af78285f370f0b346cd487c96dcf82c66f2196913073ee4d909808c89", [:make, :rebar3], [{:rabbit_common, "3.6.8", [hex: :rabbit_common, optional: false]}]}, +%{"amqp": {:hex, :amqp, "0.2.0", "ec41c4327ca7c9b1ac0baabb824d47f77a14754bed89b69bb9d32041642d59a3", [:mix], [{:amqp_client, "~> 3.6.8", [hex: :amqp_client, optional: false]}, {:rabbit_common, "~> 3.6.8", [hex: :rabbit_common, optional: false]}]}, + "amqp_client": {:hex, :amqp_client, "3.6.9", "076d7c68abc670d1bb44bbc357e4fba991a3ab53ce6160ec43576c653849a643", [:make, :rebar3], [{:rabbit_common, "3.6.9", [hex: :rabbit_common, optional: false]}]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], []}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], []}, "certifi": {:hex, :certifi, "1.0.0", "1c787a85b1855ba354f0b8920392c19aa1d06b0ee1362f9141279620a5be2039", [:rebar3], []}, - "comeonin": {:hex, :comeonin, "2.6.0", "74c288338b33205f9ce97e2117bb9a2aaab103a1811d243443d76fdb62f904ac", [:make, :mix], []}, + "comeonin": {:hex, :comeonin, "3.0.2", "8b213268a6634bd2e31a8035a963e974681d13ccc1f73f2ae664b6ac4e993c96", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, optional: false]}]}, "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, optional: false]}]}, "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], []}, - "credo": {:hex, :credo, "0.7.1", "bd09fe5bc33357aed187a405a695b586d36b4a836d4377ce496651aa483a7c18", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, optional: false]}]}, + "credo": {:hex, :credo, "0.7.2", "850463f126c09227994967fdcf8b8ad7684ab220f7727c00bcafc0ac37bd3660", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, optional: false]}]}, "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, "dialyxir": {:hex, :dialyxir, "0.5.0", "5bc543f9c28ecd51b99cc1a685a3c2a1a93216990347f259406a910cf048d1d7", [:mix], []}, "ecto": {:hex, :ecto, "2.1.4", "d1ba932813ec0e0d9db481ef2c17777f1cefb11fc90fa7c142ff354972dfba7e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, - "elastix": {:hex, :elastix, "0.3.2", "b838dfbe00010946e46392cdda0692834dc5b3b83c810c9761ac81f3e7e5416d", [:mix], [{:httpoison, "~> 0.11", [hex: :httpoison, optional: false]}, {:poison, "~> 3.1", [hex: :poison, optional: false]}]}, + "elastix": {:hex, :elastix, "0.4.0", "aa53f2fe0c7e78242cd0891ff93b5b1b2452a9a6bf9010dbc34c9896217daf57", [:mix], [{:httpoison, ">= 0.7.0", [hex: :httpoison, optional: false]}, {:poison, "~> 3.1", [hex: :poison, optional: false]}]}, + "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [:mix], []}, "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []}, "gettext": {:hex, :gettext, "0.13.1", "5e0daf4e7636d771c4c71ad5f3f53ba09a9ae5c250e1ab9c42ba9edccc476263", [:mix], []}, "guardian": {:hex, :guardian, "0.14.2", "b1db9ef172b3d47259eed3cf463321a3ed2cfadee471f271810b0e4c18b2ac22", [:mix], [{:jose, "~> 1.8", [hex: :jose, optional: false]}, {:phoenix, "~> 1.2.0", [hex: :phoenix, optional: true]}, {:plug, "~> 1.0", [hex: :plug, optional: false]}, {:poison, ">= 1.3.0", [hex: :poison, optional: false]}, {:uuid, ">=1.1.1", [hex: :uuid, optional: false]}]}, "hackney": {:hex, :hackney, "1.7.1", "e238c52c5df3c3b16ce613d3a51c7220a784d734879b1e231c9babd433ac1cb4", [:rebar3], [{:certifi, "1.0.0", [hex: :certifi, optional: false]}, {:idna, "4.0.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]}, "httpoison": {:hex, :httpoison, "0.11.1", "d06c571274c0e77b6cc50e548db3fd7779f611fbed6681fd60a331f66c143a0b", [:mix], [{:hackney, "~> 1.7.0", [hex: :hackney, optional: false]}]}, "idna": {:hex, :idna, "4.0.0", "10aaa9f79d0b12cf0def53038547855b91144f1bfcc0ec73494f38bb7b9c4961", [:rebar3], []}, - "jose": {:hex, :jose, "1.8.2", "2945d4ad6cc94515c606ade52455be3d6bba15578b7e2beed1704ad0cb1e1c80", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, optional: false]}]}, + "jose": {:hex, :jose, "1.8.3", "1285151e6f3b70aa8b60c27801aed6d20f7bb9e87116fd7d1e405ea450549aad", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, optional: false]}]}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, "mime": {:hex, :mime, "1.1.0", "01c1d6f4083d8aa5c7b8c246ade95139620ef8effb009edde934e0ec3b28090a", [:mix], []}, "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, + "passport": {:git, "https://github.com/opendrops/passport.git", "35d0df88a2eb169be724f04d0db06a2e41a5d523", []}, "phoenix": {:hex, :phoenix, "1.2.3", "b68dd6a7e6ff3eef38ad59771007d2f3f344988ea6e658e9b2c6ffb2ef494810", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, optional: false]}, {:plug, "~> 1.4 or ~> 1.3.3 or ~> 1.2.4 or ~> 1.1.8 or ~> 1.0.5", [hex: :plug, optional: false]}, {:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: false]}]}, "phoenix_ecto": {:hex, :phoenix_ecto, "3.2.3", "450c749876ff1de4a78fdb305a142a76817c77a1cd79aeca29e5fc9a6c630b26", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, optional: true]}, {:plug, "~> 1.0", [hex: :plug, optional: false]}]}, "phoenix_html": {:hex, :phoenix_html, "2.9.3", "1b5a2122cbf743aa242f54dced8a4f1cc778b8bd304f4b4c0043a6250c58e258", [:mix], [{:plug, "~> 1.0", [hex: :plug, optional: false]}]}, @@ -29,7 +31,7 @@ "plug": {:hex, :plug, "1.3.4", "b4ef3a383f991bfa594552ded44934f2a9853407899d47ecc0481777fb1906f6", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1", [hex: :cowboy, optional: true]}, {:mime, "~> 1.0", [hex: :mime, optional: false]}]}, "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []}, - "rabbit_common": {:hex, :rabbit_common, "3.6.8", "43bae839233d6ecdf8a06c1af3a985cfdb5d76232ec6e1cbe7d0469c4e5f03f5", [:make, :rebar3], []}, + "rabbit_common": {:hex, :rabbit_common, "3.6.9", "d54e1bc366f5f553a75055dfc6c93f0025efee0322f98bdf8597aff8482b996d", [:make, :rebar3], []}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}, "uuid": {:hex, :uuid, "1.1.7", "007afd58273bc0bc7f849c3bdc763e2f8124e83b957e515368c498b641f7ab69", [:mix], []}} diff --git a/package.json b/package.json index cde2993..f187a4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "microcrawler-webapp", - "version": "0.0.7", + "version": "0.0.9", "description": "Microcrawler Web Application", "repository": { "type": "git", @@ -45,7 +45,6 @@ "less": "^2.7.1", "less-loader": "^2.2.3", "mathjs": "^3.5.3", - "microcrawler-crawler-all": "0.0.18", "mocha": "^3.1.0", "moment": "^2.15.1", "node-sass": "^3.10.1", @@ -89,7 +88,6 @@ "babel-plugin-transform-react-display-name": "^6.8.0", "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-react": "^6.16.0", - "babel-preset-stage-0": "^6.16.0", "brunch": "2.8.2", "clean-css-brunch": "~2.0.0", "css-brunch": "~2.6.1", diff --git a/webpack/webpack.config.babel.js b/webpack/webpack.config.babel.js index 73aaa58..de10a70 100644 --- a/webpack/webpack.config.babel.js +++ b/webpack/webpack.config.babel.js @@ -73,10 +73,8 @@ module.exports = { __DEVTOOLS__: true // <-------- DISABLE redux-devtools HERE }), new WriteFilePlugin(), - new ExtractTextPlugin('app.css', { - allChunks: true - }), - new webpack.optimize.OccurenceOrderPlugin(), + // new ExtractTextPlugin('app.css'), + // new webpack.optimize.OccurenceOrderPlugin(), new webpack.HotModuleReplacementPlugin(), // new webpack.optimize.DedupePlugin(), // new webpack.optimize.UglifyJsPlugin({