Merge pull request #531 from wellfire/add-data

Updated P2P functionality & extra commands

Author: alexlittle

orb/management/commands/add_super_user.py

@@ -0,0 +1,51 @@
+"""
+Management command to add a superuser with CLI args including UserProfile
+
+This should only be used to programmatically create superusers with *initial*
+passwords, to be changed after creation.
+
+Usage:
+
+    django-admin.py add_super_user --email="someone@hello.com" --password="COOLPASSWORD"
+
+"""
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+from orb.models import UserProfile
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--email',
+            dest='email',
+            help='Email address',
+            required=True,
+        )
+        parser.add_argument(
+            '--password',
+            dest='password',
+            help='Initial password',
+            required=True,
+        )
+
+    def handle(self, *args, **options):
+
+        username = options.get('email')
+
+        try:
+            user = get_user_model().objects.get(username=username)
+        except get_user_model().DoesNotExist:
+            get_user_model().objects.create_superuser(
+                username,
+                password=options.get('password'),
+                email=options.get('email'),
+            )
+            UserProfile.objects.create(user=user)
+            self.stdout.write("Create new user '{}'".format(username))
+        else:
+            self.stdout.write("Found existing user '{}'".format(username))
+

orb/models.py

@@ -1,3 +1,4 @@
+import itertools
 import os
 import uuid
 
@@ -11,6 +12,9 @@
 from django.db.models import Avg, Count
 from django.utils.translation import ugettext_lazy as _
 from modeltranslation.utils import build_localized_fieldname
+from typing import Any
+from typing import Dict
+from typing import Iterable
 
 from orb import signals
 from orb.analytics.models import UserLocationVisualization
@@ -151,7 +155,9 @@ def update_from_api(self, api_data):
 
         import_user = get_import_user()
 
-        for attr, value in api_data.iteritems():
+        cleaned_api_data = clean_api_data(api_data, 'attribution', 'description', 'title')
+
+        for attr, value in cleaned_api_data.iteritems():
             setattr(self, attr, value)
 
         self.update_user = import_user
@@ -172,6 +178,7 @@ def create_from_api(cls, api_data, peer=None):
             the Resource object created
 
         """
+
         resource_files = api_data.pop('files', [])
         languages = api_data.pop('languages', [])
         tags = api_data.pop('tags', [])
@@ -181,18 +188,37 @@ def create_from_api(cls, api_data, peer=None):
 
         import_user = get_import_user()
 
-        resource = cls.resources.create(source_peer=peer, create_user=import_user, update_user=import_user, **api_data)
+        cleaned_api_data = clean_api_data(api_data, 'attribution', 'description', 'title')
+
+        resource = cls.resources.create(
+            source_peer=peer,
+            create_user=import_user,
+            update_user=import_user,
+            **cleaned_api_data
+        )
 
         ResourceURL.objects.bulk_create([
-            ResourceURL.from_url_data(resource, resource_url_data, import_user)
+            ResourceURL.from_url_data(
+                resource,
+                clean_api_data(resource_url_data, "description", "title"),
+                import_user,
+            )
             for resource_url_data in resource_urls
         ] + [
-            ResourceURL.from_file_data(resource, resource_file_data, import_user)
+            ResourceURL.from_file_data(
+                resource,
+                clean_api_data(resource_file_data, "description", "title"),
+                import_user,
+            )
             for resource_file_data in resource_files
         ])
 
         for tag_data in tags:
-            ResourceTag.create_from_api_data(resource, tag_data['tag'], user=import_user)
+            ResourceTag.create_from_api_data(
+                resource,
+                clean_api_data(tag_data['tag'], "category", "description", "name", "summary"),
+                user=import_user,
+            )
 
         return resource
 
@@ -668,6 +694,10 @@ def create_from_api_data(cls, resource, api_data, user=None):
             for field in category_fields
         }
 
+        for field in api_data.keys():
+            if field.startswith("category"):
+                del api_data[field]
+
         category, created = Category.objects.get_or_create(name=category_name, defaults=category_name_translations)
 
         api_data['category'] = category
@@ -903,3 +933,33 @@ def get_import_user():
         user.save()
         return user
 
+
+def clean_api_data(data, *fields):
+    # type: (Dict[unicode, Any], Iterable[unicode]) -> Dict[unicode, Any]
+    """
+    Returns API resource data with unusable translations filtered
+
+    Args:
+        data: original API data
+        *fields: translated field names
+
+    Returns:
+        same API data with unsupported translations dropped
+
+    """
+    language_codes = [l[0].replace('-', '_') for l in settings.LANGUAGES]
+    supported_field_translations = [
+        "{}_{}".format(field, language)
+        for (field, language) in itertools.product(fields, language_codes)
+    ]
+    def allowed_field(test_field):
+        # type: (unicode) -> bool
+        if test_field != 'id' or test_field in fields or test_field in supported_field_translations:
+            return True
+        return not any([test_field.startswith(f) for f in fields])
+
+    return {
+        key: value
+        for key, value in data.items()
+        if allowed_field(key)
+    }

orb/peers/management/commands/add_peer.py

@@ -0,0 +1,61 @@
+"""
+Management command to add a peer to the local list of peers.
+
+The command will return exit code 0 if added OR the peer already exists, and
+1 for any other error.
+
+Usage:
+
+    django-admin.py add_peer https://some.host.org --user=username --key=APIKEY --name=SomeHost
+
+"""
+
+from django.core.management.base import BaseCommand
+from django.core.validators import URLValidator
+from django.core.validators import ValidationError
+
+from orb.peers.models import Peer
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('host')
+        parser.add_argument(
+            '--name',
+            dest='name',
+            help='Name of the peer',
+            required=True,
+        )
+        parser.add_argument(
+            '--user',
+            dest='user',
+            help='API username',
+            required=True,
+        )
+        parser.add_argument(
+            '--key',
+            dest='key',
+            help='API key',
+            required=True,
+        )
+
+    def handle(self, *args, **options):
+        host = options.get('host')
+
+        try:
+            URLValidator()(host)
+        except ValidationError as err:
+            self.stderr.write("The host name '{}' is not a valid host URL".format(host))
+            exit(1)
+
+        peer, created = Peer.objects.get_or_create(host=host, defaults={
+            'name': options.get('name'),
+            'api_user': options.get('user'),
+            'api_key': options.get('key'),
+        })
+
+        if not created:
+            self.stdout.write("Found existing peer for '{}', not updating".format(host))
+        else:
+            self.stdout.write("Created new peer '{}'".format(peer))

orb/peers/models.py

@@ -90,7 +90,8 @@ def default_writer(value):
 
         total_count, resource_list = self.client.list_resources(**filters)
 
-        for api_resource in resource_list:
+        for initial_api_resource in resource_list:
+            api_resource = self.client.get_resource_by_id(initial_api_resource['id'])
             try:
                 local_resource = Resource.resources.get(guid=api_resource['guid'])
             except Resource.DoesNotExist:

orb/profiles/models.py

@@ -1,2 +1 @@
-
 # Create your models here.

requirements/base.txt

@@ -35,7 +35,8 @@ dj_database_url==0.4.2
 
 dicttoxml==1.7.4
 
-git+https://github.com/mPowering/orb-api-python.git@v0.1.1#orb_api
+git+https://github.com/mPowering/orb-api-python.git@v1.1.0#orb_api
 
 Markdown==2.6.9
 enum34==1.1.6
+typing==3.6.1