[WIP] Add uri logging to postgres normalized document

.gitignore

@@ -58,7 +58,7 @@ archive/*
 .DS_Store
 
 worker_manager/recent_files.txt
-scrapi/settings/local.py
+**/settings/local.py
 celerybeat-schedule.db
 records/
 archive/*

.travis.yml

@@ -7,13 +7,21 @@ python:
 services:
   - cassandra
   - elasticsearch
+  - postgres
+
+addons:
+  postgresql: "9.4"
 
 install:
+  - cp api/api/settings/local-travis.py api/api/settings/local.py
   - pip install -r dev-requirements.txt
   - pip install coveralls
   - cp scrapi/settings/travis-dist.py scrapi/settings/local.py
 
-before_script: flake8 .
+before_script:
+  - flake8 .
+  - psql -c "CREATE DATABASE scrapi;" -U postgres
+  - python manage.py migrate
 
 script:
   - invoke provider_map

api/api/settings/__init__.py

@@ -0,0 +1,6 @@
+from .defaults import *
+
+try:
+    from .local import *
+except ImportError as error:
+    raise ImportError("No local.py settings file found.")

api/api/settings/defaults.py

@@ -0,0 +1,121 @@
+"""
+Django settings for shareregistration project.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.7/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.7/ref/settings/
+"""
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+
+from django.conf import global_settings
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '...'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+TEMPLATE_DEBUG = True
+
+DOMAIN = 'http://localhost:8000'
+
+ALLOWED_HOSTS = [
+    '.osf.io',
+    'testserver'
+]
+
+# Application definition
+
+INSTALLED_APPS = (
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.contenttypes',
+    'django.contrib.sites',
+    'api.webview',
+    'rest_framework',
+    'corsheaders'
+)
+
+SITE_ID = 1
+
+ACCOUNT_ACTIVATION_DAYS = 7
+
+MIDDLEWARE_CLASSES = (
+    'corsheaders.middleware.CorsMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+)
+
+TEMPLATE_CONTEXT_PROCESSORS = global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
+    'django.core.context_processors.debug',
+    'django.contrib.auth.context_processors.auth',
+)
+
+ROOT_URLCONF = 'api.api.urls'
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'scrapi'
+    }
+}
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.7/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'EST'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.7/howto/static-files/
+
+STATICFILES_DIRS = (
+    os.path.join(
+        os.path.dirname(__file__),
+        '..',  # up one level from the settings directory
+        'static'
+    ),
+)
+
+STATIC_ROOT = os.path.join('..', BASE_DIR, 'static')
+STATIC_URL = '{}/static/'.format(DOMAIN)
+
+TEMPLATE_DIRS = [os.path.join(BASE_DIR, 'templates')]
+
+REST_FRAMEWORK = {
+    'PAGE_SIZE': 10,
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        'rest_framework.authentication.BasicAuthentication',
+        'rest_framework.authentication.SessionAuthentication',
+        'rest_framework.authentication.TokenAuthentication',
+    )
+}
+
+CORS_ORIGIN_WHITELIST = (
+    'osf.io',
+    'staging.osf.io'
+)

api/api/settings/local-dist.py

@@ -0,0 +1,25 @@
+SECRET_KEY = 'My Secret Key'
+
+DEBUG = True
+
+DOMAIN = 'http://localhost:8000'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'scrapi',
+        # 'USER': 'name',
+        # 'PASSWORD': 'password',
+        # 'HOST': '127.0.0.1',
+        # 'PORT': '5432'
+    }
+}
+
+STATIC_URL = '/static/'
+
+
+CORS_ORIGIN_WHITELIST = (
+    'localhost:5000',
+    'osf.io',
+    'staging.osf.io'
+)

api/api/settings/local-travis.py

@@ -0,0 +1,6 @@
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'scrapi',
+    }
+}

api/api/static/css/api_custom.css

@@ -0,0 +1,64 @@
+/* custom navigation styles */
+.navbar {
+  width: 100%;
+  position: fixed;
+  left: 0;
+  top: 0;
+  z-index: 3;
+}
+
+.navbar {
+  background: #BBBBBB;
+  color: white;
+  border: none;
+  border-top: 5px solid #00B3EA;
+  border-radius: 0px;
+}
+
+.navbar .nav li, .navbar .nav li a, .navbar .brand:hover {
+  color: white;
+}
+
+.nav-list > .active > a, .nav-list > .active > a:hover {
+  background: #105B74;
+}
+
+.navbar .dropdown-menu li a, .navbar .dropdown-menu li {
+  color: #21B6E8;
+}
+
+.navbar .dropdown-menu li a:hover {
+  background: #EEEEEE;
+  color: #197D9E;
+}
+
+.pagination>.disabled>a,
+.pagination>.disabled>a:hover,
+.pagination>.disabled>a:focus {
+  cursor: not-allowed;
+  pointer-events: none;
+}
+
+.pager>.disabled>a,
+.pager>.disabled>a:hover,
+.pager>.disabled>a:focus {
+  pointer-events: none;
+}
+
+.pager .next {
+  margin-left: 10px;
+}
+
+.logo {
+  width: 60px;
+  padding-top: 5px;
+  padding-bottom: 5px;
+}
+
+.str, .atv {
+  color: #5D615B;
+}
+
+body a {
+  color: #20B3E4;
+}

api/api/urls.py

@@ -0,0 +1,22 @@
+"""api URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.8/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Add an import:  from blog import urls as blog_urls
+    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
+"""
+from django.conf.urls import include, url
+from django.contrib import admin
+
+urlpatterns = [
+    url(r'^admin/', include(admin.site.urls)),
+    url(r'^', include('api.webview.urls')),
+]

api/api/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for api project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.settings")
+
+application = get_wsgi_application()

api/templates/base.html

@@ -0,0 +1,30 @@
+{% load staticfiles %}
+<html>
+    <head>
+        <link href="{% static 'bower_components/bootstrap/dist/css/bootstrap.min.css' %}" rel="stylesheet" media="screen"/>
+        <link href="{% static 'css/custom.css' %}" rel="stylesheet" media="screen"/>
+    </head>
+
+    <body>
+        <div class="container">
+            <div class="jumbotron">
+                <img class="logo_jumbotron" src="{% static 'img/share.png' %}"/>
+
+                {% block header_content %}
+                {% endblock %}
+            </div>
+            <div class="help_information">
+                    Questions or feedback? Contact us at <a href="mailto:share-support@osf.io">share-support@osf.io</a>
+            </div>
+            {% block content %}
+            {% endblock %}
+        </div>
+
+        <script src="{% static 'bower_components/jquery/dist/jquery.min.js' %}"></script>
+        <script src="{% static 'bower_components/bootstrap/dist/js/bootstrap.min.js' %}"></script>
+        <script src="{% static 'bower_components/pym.js/dist/pym.min.js' %}"></script>
+        <script>new pym.Child({ id: 'share_registration_iframe' });</script>
+        {% block javascript_bottom %}
+        {% endblock %}
+    </body>
+</html>

api/templates/rest_framework/api.html

@@ -0,0 +1,29 @@
+{% extends "rest_framework/base.html" %}
+    {% load url from future %}
+    {% load rest_framework %}
+    {% load static %}
+
+    {% block title %}SHARE Notify{% endblock %}
+
+    {% block style %}
+        {% block bootstrap_theme %}
+            <link rel="stylesheet" type="text/css" href="{% static "rest_framework/css/bootstrap.min.css" %}"/>
+            <link rel="stylesheet" type="text/css" href="{% static "rest_framework/css/bootstrap-tweaks.css" %}"/>
+        {% endblock %}
+        <link rel="stylesheet" type="text/css" href="{% static "rest_framework/css/prettify.css" %}"/>
+        <link rel="stylesheet" type="text/css" href="{% static "rest_framework/css/default.css" %}"/>
+        <link rel="stylesheet" type="text/css" href="{% static "css/api_custom.css" %}"/>
+    {% endblock %}
+
+    {% block branding %}
+        <img class="logo" src="{% static 'img/share.png' %}"/>
+    {% endblock %}
+
+    {% block userlinks %}
+    {{ block.super }}
+    {% if user.is_authenticated %}
+        <li><a href="{% url 'get-api-key' %}">Get API Key</a></li>
+    {% endif %}
+    {% endblock %}
+
+

api/webview/models.py

@@ -0,0 +1,13 @@
+from django.db import models
+
+from django_pgjson.fields import JsonField
+
+
+class Document(models.Model):
+    source = models.CharField(max_length=100)
+    docID = models.CharField(max_length=100)
+
+    providerUpdatedDateTime = models.DateTimeField(null=True)
+
+    raw = JsonField()
+    normalized = JsonField()

api/webview/serializers.py

@@ -0,0 +1,10 @@
+from rest_framework import serializers
+
+from api.webview.models import Document
+
+
+class DocumentSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = Document
+        fields = ('id', 'providerUpdatedDateTime', 'source', 'docID', 'raw', 'normalized')