From 501536224e1a65a658cf6ad43e92df38a13a25f5 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Jul 2016 11:48:27 +0100 Subject: [PATCH 01/51] Switch to the django-1.7 branch of popit-django --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index afb9f3b3..af16ec05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ mock==1.0.1 ndg-httpsclient==0.4.1 nose==1.3.7 oauthlib==1.1.2 --e git://github.com/mysociety/popit-django.git@94accb3fc38d2205f298e7177f6f1a206707288b#egg=popit_django +-e git://github.com/mysociety/popit-django.git@django-1.7#egg=popit-django pyasn1==0.1.9 pycparser==2.14 PyJWT==1.4.0 From 17b0866f1c622f12c3db1d5c1600868d8743912e Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 7 Jul 2016 11:26:44 +0100 Subject: [PATCH 02/51] Add mysociety-django-popolo at version 0.0.7 --- requirements.txt | 1 + writeit/settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index af16ec05..4662ed63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ kombu==3.0.35 libsass==0.11.1 markdown2==2.3.1 mock==1.0.1 +mysociety-django-popolo==0.0.7 ndg-httpsclient==0.4.1 nose==1.3.7 oauthlib==1.1.2 diff --git a/writeit/settings.py b/writeit/settings.py index 48104f92..e1ca96f1 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -212,6 +212,7 @@ 'djangoplugins', 'pagination', 'popit', + 'popolo', 'contactos', 'mailit', 'tastypie', From 4ce0c9dd210e558138f29124a8d8c1a12d4d9f2c Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Sun, 18 Sep 2016 16:43:15 +0100 Subject: [PATCH 03/51] Add multiple-django-popolo-sources --- requirements.txt | 1 + writeit/settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 4662ed63..3963ceb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ ndg-httpsclient==0.4.1 nose==1.3.7 oauthlib==1.1.2 -e git://github.com/mysociety/popit-django.git@django-1.7#egg=popit-django +multiple-django-popolo-sources==0.0.3 pyasn1==0.1.9 pycparser==2.14 PyJWT==1.4.0 diff --git a/writeit/settings.py b/writeit/settings.py index e1ca96f1..276b81c3 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -213,6 +213,7 @@ 'pagination', 'popit', 'popolo', + 'popolo_sources', 'contactos', 'mailit', 'tastypie', From 73af672ca5a84fe2c820ad50a4cde9c0f504de6c Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 7 Jul 2016 22:30:55 +0100 Subject: [PATCH 04/51] Add parallel fields / models for migrating data to django-popolo --- .../migrations/0002_contact_popolo_person.py | 21 ++++++++++ contactos/models.py | 2 + .../0003_add_parallel_popolo_data.py | 39 +++++++++++++++++++ instance/models.py | 11 ++++++ .../0003_add_parallel_popolo_data.py | 27 +++++++++++++ nuntium/models.py | 3 ++ 6 files changed, 103 insertions(+) create mode 100644 contactos/migrations/0002_contact_popolo_person.py create mode 100644 instance/migrations/0003_add_parallel_popolo_data.py create mode 100644 nuntium/migrations/0003_add_parallel_popolo_data.py diff --git a/contactos/migrations/0002_contact_popolo_person.py b/contactos/migrations/0002_contact_popolo_person.py new file mode 100644 index 00000000..fff5e800 --- /dev/null +++ b/contactos/migrations/0002_contact_popolo_person.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('popolo', '0002_update_models_from_upstream'), + ('contactos', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='contact', + name='popolo_person', + field=models.ForeignKey(blank=True, to='popolo.Person', null=True), + preserve_default=True, + ), + ] diff --git a/contactos/models.py b/contactos/models.py index 32d9b0f0..af818e70 100644 --- a/contactos/models.py +++ b/contactos/models.py @@ -1,5 +1,6 @@ from django.db import models from popit.models import Person +from popolo.models import Person as PopoloPerson from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User from django.db.models.signals import pre_save @@ -22,6 +23,7 @@ class Contact(models.Model): """docstring for Contact""" contact_type = models.ForeignKey('ContactType') person = models.ForeignKey(Person) + popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) value = models.CharField(max_length=512) is_bounced = models.BooleanField(default=False) owner = models.ForeignKey(User, related_name="contacts", null=True) diff --git a/instance/migrations/0003_add_parallel_popolo_data.py b/instance/migrations/0003_add_parallel_popolo_data.py new file mode 100644 index 00000000..a0b43b6a --- /dev/null +++ b/instance/migrations/0003_add_parallel_popolo_data.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('popolo', '0002_update_models_from_upstream'), + ('instance', '0001_initial'), + ('popolo_sources', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='InstanceMembership', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('person', models.ForeignKey(to='popolo.Person')), + ('writeitinstance', models.ForeignKey(to='instance.WriteItInstance')), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.AddField( + model_name='writeitinstance', + name='popolo_persons', + field=models.ManyToManyField(related_name='writeit_instances', through='instance.InstanceMembership', to='popolo.Person'), + preserve_default=True, + ), + migrations.AddField( + model_name='writeitinstancepopitinstancerecord', + name='popolo_source', + field=models.ForeignKey(blank=True, to='popolo_sources.PopoloSource', null=True), + preserve_default=True, + ), + ] diff --git a/instance/models.py b/instance/models.py index b3638a81..16e1cc9a 100644 --- a/instance/models.py +++ b/instance/models.py @@ -11,6 +11,8 @@ from autoslug import AutoSlugField from nuntium.popit_api_instance import PopitApiInstance from popit.models import Person, ApiInstance +from popolo.models import Person as PopoloPerson +from popolo_sources.models import PopoloSource from requests.exceptions import ConnectionError from subdomains.utils import reverse @@ -24,6 +26,9 @@ class WriteItInstance(models.Model): persons = models.ManyToManyField(Person, related_name='writeit_instances', through='Membership') + popolo_persons = models.ManyToManyField(PopoloPerson, + related_name='writeit_instances', + through='InstanceMembership') owner = models.ForeignKey(User, related_name="writeitinstances") def add_person(self, person): @@ -115,6 +120,11 @@ class Membership(models.Model): writeitinstance = models.ForeignKey(WriteItInstance) +class InstanceMembership(models.Model): + person = models.ForeignKey(PopoloPerson) + writeitinstance = models.ForeignKey(WriteItInstance) + + def new_write_it_instance(sender, instance, created, **kwargs): from nuntium.models import ( NewAnswerNotificationTemplate, ConfirmationTemplate) @@ -147,6 +157,7 @@ class WriteitInstancePopitInstanceRecord(models.Model): ) writeitinstance = models.ForeignKey(WriteItInstance) popitapiinstance = models.ForeignKey(ApiInstance) + popolo_source = models.ForeignKey(PopoloSource, null=True, blank=True) periodicity = models.CharField( max_length="2", choices=PERIODICITY, diff --git a/nuntium/migrations/0003_add_parallel_popolo_data.py b/nuntium/migrations/0003_add_parallel_popolo_data.py new file mode 100644 index 00000000..bc954c1a --- /dev/null +++ b/nuntium/migrations/0003_add_parallel_popolo_data.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('popolo', '0002_update_models_from_upstream'), + ('nuntium', '0002_fix_bad_datetime_defaults'), + ] + + operations = [ + migrations.AddField( + model_name='answer', + name='popolo_person', + field=models.ForeignKey(blank=True, to='popolo.Person', null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='nocontactom', + name='popolo_person', + field=models.ForeignKey(blank=True, to='popolo.Person', null=True), + preserve_default=True, + ), + ] diff --git a/nuntium/models.py b/nuntium/models.py index 27e2df9e..8356ae91 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -6,6 +6,7 @@ from django.utils.translation import override, ugettext_lazy as _ from django.core.exceptions import ValidationError, ObjectDoesNotExist from popit.models import Person +from popolo.models import Person as PopoloPerson from contactos.models import Contact from .plugins import OutputPlugin from django.contrib.contenttypes.models import ContentType @@ -318,6 +319,7 @@ class Answer(models.Model): content = models.TextField() content_html = models.TextField() person = models.ForeignKey(Person) + popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) message = models.ForeignKey(Message, related_name='answers') created = models.DateTimeField(auto_now=True, null=True) @@ -436,6 +438,7 @@ class Meta: class NoContactOM(AbstractOutboundMessage): person = models.ForeignKey(Person) + popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) # This will happen everytime a contact is created From 8231279411ac457c09b7aceccb45bec44cd8034f Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 11 Jul 2016 12:20:35 +0100 Subject: [PATCH 05/51] Migrate data from popit-django to django-popolo --- .../0004_add_django_popolo_people.py | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 instance/migrations/0004_add_django_popolo_people.py diff --git a/instance/migrations/0004_add_django_popolo_people.py b/instance/migrations/0004_add_django_popolo_people.py new file mode 100644 index 00000000..83121022 --- /dev/null +++ b/instance/migrations/0004_add_django_popolo_people.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import re +from urlparse import urlsplit, urlunsplit + +from django.db import migrations +from django.contrib.contenttypes.management import update_contenttypes + + +def is_proxy_url(url): + return url == 'everypolitician-writeinpublic.herokuapp.com' + + +def update_source_url(original_url): + split_url = urlsplit(original_url) + has_popit_in_domain = 'popit' in split_url.netloc.split('.') + is_possible_api_url = re.match(r'/?$|^/api', split_url.path) + if is_proxy_url(split_url.netloc): + # If this is one of the faked PopIt instances for + # EveryPolitician, change that to the EveryPolitican JSON. + country, house = split_url.path.lstrip('/').split('/')[:2] + url = ('https://raw.githubusercontent.com' + '/everypolitician/everypolitician-data/master/data/' + '{country}/{house}/ep-popolo-v1.0.json').format( + country=country, house=house) + elif has_popit_in_domain and is_possible_api_url: + # If looks like a PopIt instance, then replace that with a + # link to the PopIt instance's export.json: + split_as_list = list(split_url) + split_as_list[2] = '/api/v0.1/export.json' + split_as_list[3] = '' + split_as_list[4] = '' + url = urlunsplit(split_as_list) + else: + url = original_url + return url + + +def forwards(apps, schema_editor): + # Make sure the content types for django-popolo exist, with a + # hacky workaround from: http://stackoverflow.com/a/35353170/223092 + popolo_app = apps.app_configs['popolo'] + popolo_app.models_module = popolo_app.models_module or True + update_contenttypes(popolo_app, verbosity=1, interactive=False) + ContentType = apps.get_model('contenttypes', 'ContentType') + person_content_type = ContentType.objects.get( + app_label='popolo', model='person') + # Create a PopoloSource for each old APIInstance + ApiInstance = apps.get_model('popit', 'ApiInstance') + PopoloSource = apps.get_model('popolo_sources', 'PopoloSource') + LinkToPopoloSource = apps.get_model('popolo_sources', 'LinkToPopoloSource') + ai_to_ps = {} + for ai in ApiInstance.objects.all(): + url = update_source_url(ai.url) + ps = PopoloSource.objects.create(url=url) + ai_to_ps[ai] = ps + # Now create a django-popolo Person for each old django-popit + # Person: + PopItPerson = apps.get_model('popit', 'Person') + PopoloPerson = apps.get_model('popolo', 'Person') + Identifier = apps.get_model('popolo', 'Identifier') + p_old_to_p_new = {} + for popit_person in PopItPerson.objects.all(): + # The popit_ids from + # everypolitician-writeinpublic.herokuapp.com sources have + # 'person/' prepended, which they won't have when we switch to + # using the upstream EveryPolitician Popolo JSON. So if it's + # from such a source, remove that prefix. + new_popolo_person_id = popit_person.popit_id + if is_proxy_url(urlsplit(popit_person.popit_url).netloc): + prefix = 'person/' + if new_popolo_person_id.startswith(prefix): + new_popolo_person_id = new_popolo_person_id[len(prefix):] + # Create the new django-popolo Person object: + new_person = PopoloPerson.objects.create( + name=popit_person.name, + summary=popit_person.summary, + image=popit_person.image, + ) + p_old_to_p_new[popit_person] = new_person + # The popit_id Identifier is to preserve the old popit_id that + # was on a popit-django Person. This will still be returned by + # the API for people that have it, and will be looked up when + # used in some URLs to avoid breaking old links. There is one + # PopIt instance referred to in the database which has NULL + # popit_id values, which would break the non-NULL constraint + # on `identifier`, so only create this if it popit_id is + # non-NULL. (Some other identifier creation, below, needs to + # be skipped in those cases too.) + if popit_person.popit_id: + Identifier.objects.create( + scheme='popit_id', + identifier=popit_person.popit_id, + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # The popit_url Identifier is to preserver the old popit_url + # that was on a popit-django Person. This should still be used + # as the canonical identifier for people in the API if they + # have it, but for newer people popolo_uri will be used. + Identifier.objects.create( + scheme='popit_url', + identifier=popit_person.popit_url, + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # The popolo:person Identifier represents the 'id' of the + # Person in the Popolo JSON source; it's required (among other + # things) so that when updating from the Popolo JSON source, + # we can tell whether a person already exists in the database + # or not. (When fetching new people, they will have it added + # as well.) + if new_popolo_person_id: + Identifier.objects.create( + scheme='popolo:person', + identifier=new_popolo_person_id, + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # The popit_django_person_id Identifier preserved the old ID + # of the popit-django Person object; this shouldn't be needed + # anywhere, but it would be foolish not to preserve it in case + # it's helpful for debugging, etc. + Identifier.objects.create( + scheme='popit_django_person_id', + identifier=popit_person.id, + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # This is the new canonical URI for the source of information + # about a person, which is used in API responses where there + # is no legacy popit_url. + if popit_person.popit_id: + Identifier.objects.create( + scheme='popolo_uri', + identifier=(ps.url + '#person-' + popit_person.popit_id), + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # Set the right PopoloSource for the person: + ps = ai_to_ps[popit_person.api_instance] + LinkToPopoloSource.objects.create( + popolo_source=ps, + deleted_from_source=False, + object_id=new_person.id, + content_type_id=person_content_type.id, + ) + # Update the parallel popolo_person attribute on Contact + Contact = apps.get_model('contactos', 'Contact') + for c in Contact.objects.all(): + c.popolo_person = p_old_to_p_new[c.person] + c.save() + # Create the parallel InstanceMembership relation: + Membership = apps.get_model('instance', 'Membership') + InstanceMembership = apps.get_model('instance', 'InstanceMembership') + for m in Membership.objects.all(): + popolo_person = p_old_to_p_new[m.person] + InstanceMembership.objects.create( + person=popolo_person, + writeitinstance=m.writeitinstance, + ) + # Update the parallel popolo_source field on + # WriteitInstancePopitInstanceRecord. + WriteitInstancePopitInstanceRecord = apps.get_model( + 'instance', 'WriteitInstancePopitInstanceRecord') + for wiipir in WriteitInstancePopitInstanceRecord.objects.all(): + wiipir.popolo_source = ai_to_ps[wiipir.popitapiinstance] + wiipir.save() + # Update the parallel popolo_person attribute on Answer: + Answer = apps.get_model('nuntium', 'Answer') + for a in Answer.objects.all(): + a.popolo_person = p_old_to_p_new[a.person] + a.save() + # Update the parallel popolo_person field on NoContactOM: + NoContactOM = apps.get_model('nuntium', 'NoContactOM') + for ncom in NoContactOM.objects.all(): + ncom.popolo_person = p_old_to_p_new[ncom.person] + ncom.save() + +def backwards(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('popit', '0001_initial'), + ('popolo_sources', '0001_initial'), + ('instance', '0003_add_parallel_popolo_data'), + ('popolo', '0002_update_models_from_upstream'), + ('contactos', '0002_contact_popolo_person'), + ('nuntium', '0003_add_parallel_popolo_data'), + ] + + operations = [ + migrations.RunPython(forwards, backwards) + ] From 197bb37676b9d231295255524d441921b2872595 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Jul 2016 14:00:40 +0100 Subject: [PATCH 06/51] After the data migration, remove the null option for new fields --- .../migrations/0003_denull_popolo_people.py | 21 +++++++++++++++ contactos/models.py | 2 +- .../migrations/0005_denull_popolo_source.py | 20 ++++++++++++++ instance/models.py | 2 +- .../migrations/0003_denull_popolo_people.py | 27 +++++++++++++++++++ nuntium/models.py | 4 +-- 6 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 contactos/migrations/0003_denull_popolo_people.py create mode 100644 instance/migrations/0005_denull_popolo_source.py create mode 100644 nuntium/migrations/0003_denull_popolo_people.py diff --git a/contactos/migrations/0003_denull_popolo_people.py b/contactos/migrations/0003_denull_popolo_people.py new file mode 100644 index 00000000..d74b75b9 --- /dev/null +++ b/contactos/migrations/0003_denull_popolo_people.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('contactos', '0002_contact_popolo_person'), + ('instance', '0004_add_django_popolo_people'), + ] + + operations = [ + migrations.AlterField( + model_name='contact', + name='popolo_person', + field=models.ForeignKey(to='popolo.Person'), + preserve_default=True, + ), + ] diff --git a/contactos/models.py b/contactos/models.py index af818e70..1821318c 100644 --- a/contactos/models.py +++ b/contactos/models.py @@ -23,7 +23,7 @@ class Contact(models.Model): """docstring for Contact""" contact_type = models.ForeignKey('ContactType') person = models.ForeignKey(Person) - popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) + popolo_person = models.ForeignKey(PopoloPerson) value = models.CharField(max_length=512) is_bounced = models.BooleanField(default=False) owner = models.ForeignKey(User, related_name="contacts", null=True) diff --git a/instance/migrations/0005_denull_popolo_source.py b/instance/migrations/0005_denull_popolo_source.py new file mode 100644 index 00000000..35ed301e --- /dev/null +++ b/instance/migrations/0005_denull_popolo_source.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('instance', '0004_add_django_popolo_people'), + ] + + operations = [ + migrations.AlterField( + model_name='writeitinstancepopitinstancerecord', + name='popolo_source', + field=models.ForeignKey(to='popolo_sources.PopoloSource'), + preserve_default=True, + ), + ] diff --git a/instance/models.py b/instance/models.py index 16e1cc9a..e137479e 100644 --- a/instance/models.py +++ b/instance/models.py @@ -157,7 +157,7 @@ class WriteitInstancePopitInstanceRecord(models.Model): ) writeitinstance = models.ForeignKey(WriteItInstance) popitapiinstance = models.ForeignKey(ApiInstance) - popolo_source = models.ForeignKey(PopoloSource, null=True, blank=True) + popolo_source = models.ForeignKey(PopoloSource) periodicity = models.CharField( max_length="2", choices=PERIODICITY, diff --git a/nuntium/migrations/0003_denull_popolo_people.py b/nuntium/migrations/0003_denull_popolo_people.py new file mode 100644 index 00000000..be3395be --- /dev/null +++ b/nuntium/migrations/0003_denull_popolo_people.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('nuntium', '0003_add_parallel_popolo_data'), + ('instance', '0004_add_django_popolo_people'), + ] + + operations = [ + migrations.AlterField( + model_name='answer', + name='popolo_person', + field=models.ForeignKey(to='popolo.Person'), + preserve_default=True, + ), + migrations.AlterField( + model_name='nocontactom', + name='popolo_person', + field=models.ForeignKey(to='popolo.Person'), + preserve_default=True, + ), + ] diff --git a/nuntium/models.py b/nuntium/models.py index 8356ae91..6a6e589d 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -319,7 +319,7 @@ class Answer(models.Model): content = models.TextField() content_html = models.TextField() person = models.ForeignKey(Person) - popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) + popolo_person = models.ForeignKey(PopoloPerson) message = models.ForeignKey(Message, related_name='answers') created = models.DateTimeField(auto_now=True, null=True) @@ -438,7 +438,7 @@ class Meta: class NoContactOM(AbstractOutboundMessage): person = models.ForeignKey(Person) - popolo_person = models.ForeignKey(PopoloPerson, null=True, blank=True) + popolo_person = models.ForeignKey(PopoloPerson) # This will happen everytime a contact is created From 7a2c004c1234aa3c96247dc31025567b0b9a8679 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Jul 2016 15:07:07 +0100 Subject: [PATCH 07/51] Remove old popit-django model references --- .../migrations/0004_remove_contact_person.py | 18 ++++++++++++ contactos/models.py | 6 ++-- instance/migrations/0006_remove_old_fields.py | 29 +++++++++++++++++++ instance/models.py | 8 ----- nuntium/admin.py | 2 +- nuntium/migrations/0004_remove_old_fields.py | 22 ++++++++++++++ nuntium/models.py | 2 -- 7 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 contactos/migrations/0004_remove_contact_person.py create mode 100644 instance/migrations/0006_remove_old_fields.py create mode 100644 nuntium/migrations/0004_remove_old_fields.py diff --git a/contactos/migrations/0004_remove_contact_person.py b/contactos/migrations/0004_remove_contact_person.py new file mode 100644 index 00000000..0144b776 --- /dev/null +++ b/contactos/migrations/0004_remove_contact_person.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('contactos', '0003_denull_popolo_people'), + ] + + operations = [ + migrations.RemoveField( + model_name='contact', + name='person', + ), + ] diff --git a/contactos/models.py b/contactos/models.py index 1821318c..679fe404 100644 --- a/contactos/models.py +++ b/contactos/models.py @@ -1,6 +1,5 @@ from django.db import models -from popit.models import Person -from popolo.models import Person as PopoloPerson +from popolo.models import Person as Person from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User from django.db.models.signals import pre_save @@ -22,8 +21,7 @@ def __unicode__(self): class Contact(models.Model): """docstring for Contact""" contact_type = models.ForeignKey('ContactType') - person = models.ForeignKey(Person) - popolo_person = models.ForeignKey(PopoloPerson) + popolo_person = models.ForeignKey(Person) value = models.CharField(max_length=512) is_bounced = models.BooleanField(default=False) owner = models.ForeignKey(User, related_name="contacts", null=True) diff --git a/instance/migrations/0006_remove_old_fields.py b/instance/migrations/0006_remove_old_fields.py new file mode 100644 index 00000000..f8d3f68c --- /dev/null +++ b/instance/migrations/0006_remove_old_fields.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('instance', '0005_denull_popolo_source'), + ] + + operations = [ + migrations.RemoveField( + model_name='membership', + name='person', + ), + migrations.RemoveField( + model_name='membership', + name='writeitinstance', + ), + migrations.RemoveField( + model_name='writeitinstance', + name='persons', + ), + migrations.DeleteModel( + name='Membership', + ), + ] diff --git a/instance/models.py b/instance/models.py index e137479e..d8600a2d 100644 --- a/instance/models.py +++ b/instance/models.py @@ -23,9 +23,6 @@ class WriteItInstance(models.Model): name = models.CharField(max_length=255) description = models.CharField(max_length=512, blank=True) slug = AutoSlugField(populate_from='name', unique=True) - persons = models.ManyToManyField(Person, - related_name='writeit_instances', - through='Membership') popolo_persons = models.ManyToManyField(PopoloPerson, related_name='writeit_instances', through='InstanceMembership') @@ -115,11 +112,6 @@ def __unicode__(self): return self.name -class Membership(models.Model): - person = models.ForeignKey(Person) - writeitinstance = models.ForeignKey(WriteItInstance) - - class InstanceMembership(models.Model): person = models.ForeignKey(PopoloPerson) writeitinstance = models.ForeignKey(WriteItInstance) diff --git a/nuntium/admin.py b/nuntium/admin.py index b0a5eb02..ba4aa77c 100644 --- a/nuntium/admin.py +++ b/nuntium/admin.py @@ -16,7 +16,7 @@ class PersonInline(admin.TabularInline): class MembershipInline(admin.TabularInline): - model = WriteItInstance.persons.through + model = WriteItInstance.popolo_persons.through class NewAnswerNotificationTemplateAdmin(admin.TabularInline): diff --git a/nuntium/migrations/0004_remove_old_fields.py b/nuntium/migrations/0004_remove_old_fields.py new file mode 100644 index 00000000..dcbcde16 --- /dev/null +++ b/nuntium/migrations/0004_remove_old_fields.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('nuntium', '0003_denull_popolo_people'), + ] + + operations = [ + migrations.RemoveField( + model_name='answer', + name='person', + ), + migrations.RemoveField( + model_name='nocontactom', + name='person', + ), + ] diff --git a/nuntium/models.py b/nuntium/models.py index 6a6e589d..6ed08554 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -318,7 +318,6 @@ def slugify_message(sender, instance, **kwargs): class Answer(models.Model): content = models.TextField() content_html = models.TextField() - person = models.ForeignKey(Person) popolo_person = models.ForeignKey(PopoloPerson) message = models.ForeignKey(Message, related_name='answers') created = models.DateTimeField(auto_now=True, null=True) @@ -437,7 +436,6 @@ class Meta: class NoContactOM(AbstractOutboundMessage): - person = models.ForeignKey(Person) popolo_person = models.ForeignKey(PopoloPerson) From 10f61f7a87f98aee81c3cad2e4187bb40a242221 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Jul 2016 15:15:13 +0100 Subject: [PATCH 08/51] Rename the now-migrated fields to the more concise names --- .../migrations/0005_rename_migrated_fields.py | 19 ++++++++++++++ contactos/models.py | 2 +- .../migrations/0007_rename_migrated_fields.py | 19 ++++++++++++++ instance/models.py | 2 +- nuntium/admin.py | 2 +- .../migrations/0005_rename_migrated_fields.py | 26 +++++++++++++++++++ nuntium/models.py | 4 +-- 7 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 contactos/migrations/0005_rename_migrated_fields.py create mode 100644 instance/migrations/0007_rename_migrated_fields.py create mode 100644 nuntium/migrations/0005_rename_migrated_fields.py diff --git a/contactos/migrations/0005_rename_migrated_fields.py b/contactos/migrations/0005_rename_migrated_fields.py new file mode 100644 index 00000000..bb0bbce7 --- /dev/null +++ b/contactos/migrations/0005_rename_migrated_fields.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('contactos', '0004_remove_contact_person'), + ] + + operations = [ + migrations.RenameField( + model_name='contact', + old_name='popolo_person', + new_name='person', + ), + ] diff --git a/contactos/models.py b/contactos/models.py index 679fe404..e5092e9f 100644 --- a/contactos/models.py +++ b/contactos/models.py @@ -21,7 +21,7 @@ def __unicode__(self): class Contact(models.Model): """docstring for Contact""" contact_type = models.ForeignKey('ContactType') - popolo_person = models.ForeignKey(Person) + person = models.ForeignKey(Person) value = models.CharField(max_length=512) is_bounced = models.BooleanField(default=False) owner = models.ForeignKey(User, related_name="contacts", null=True) diff --git a/instance/migrations/0007_rename_migrated_fields.py b/instance/migrations/0007_rename_migrated_fields.py new file mode 100644 index 00000000..fa9b0ed8 --- /dev/null +++ b/instance/migrations/0007_rename_migrated_fields.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('instance', '0006_remove_old_fields'), + ] + + operations = [ + migrations.RenameField( + model_name='writeitinstance', + old_name='popolo_persons', + new_name='persons', + ), + ] diff --git a/instance/models.py b/instance/models.py index d8600a2d..874ec0ac 100644 --- a/instance/models.py +++ b/instance/models.py @@ -23,7 +23,7 @@ class WriteItInstance(models.Model): name = models.CharField(max_length=255) description = models.CharField(max_length=512, blank=True) slug = AutoSlugField(populate_from='name', unique=True) - popolo_persons = models.ManyToManyField(PopoloPerson, + persons = models.ManyToManyField(PopoloPerson, related_name='writeit_instances', through='InstanceMembership') owner = models.ForeignKey(User, related_name="writeitinstances") diff --git a/nuntium/admin.py b/nuntium/admin.py index ba4aa77c..b0a5eb02 100644 --- a/nuntium/admin.py +++ b/nuntium/admin.py @@ -16,7 +16,7 @@ class PersonInline(admin.TabularInline): class MembershipInline(admin.TabularInline): - model = WriteItInstance.popolo_persons.through + model = WriteItInstance.persons.through class NewAnswerNotificationTemplateAdmin(admin.TabularInline): diff --git a/nuntium/migrations/0005_rename_migrated_fields.py b/nuntium/migrations/0005_rename_migrated_fields.py new file mode 100644 index 00000000..2501b78f --- /dev/null +++ b/nuntium/migrations/0005_rename_migrated_fields.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import datetime +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('nuntium', '0004_remove_old_fields'), + ] + + operations = [ + migrations.RenameField( + model_name='answer', + old_name='popolo_person', + new_name='person', + ), + migrations.RenameField( + model_name='nocontactom', + old_name='popolo_person', + new_name='person', + ), + ] diff --git a/nuntium/models.py b/nuntium/models.py index 6ed08554..1e592dd1 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -318,7 +318,7 @@ def slugify_message(sender, instance, **kwargs): class Answer(models.Model): content = models.TextField() content_html = models.TextField() - popolo_person = models.ForeignKey(PopoloPerson) + person = models.ForeignKey(PopoloPerson) message = models.ForeignKey(Message, related_name='answers') created = models.DateTimeField(auto_now=True, null=True) @@ -436,7 +436,7 @@ class Meta: class NoContactOM(AbstractOutboundMessage): - popolo_person = models.ForeignKey(PopoloPerson) + person = models.ForeignKey(PopoloPerson) # This will happen everytime a contact is created From 4a5c89ec665f5e9f864ffcec5c0477533f8497c9 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 14 Jul 2016 12:28:59 +0100 Subject: [PATCH 09/51] Remove a now unneeded popitapiinstance field --- ...ancepopitinstancerecord_popitapiinstance.py | 18 ++++++++++++++++++ instance/models.py | 1 - 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 instance/migrations/0008_remove_writeitinstancepopitinstancerecord_popitapiinstance.py diff --git a/instance/migrations/0008_remove_writeitinstancepopitinstancerecord_popitapiinstance.py b/instance/migrations/0008_remove_writeitinstancepopitinstancerecord_popitapiinstance.py new file mode 100644 index 00000000..6adc3a0a --- /dev/null +++ b/instance/migrations/0008_remove_writeitinstancepopitinstancerecord_popitapiinstance.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('instance', '0007_rename_migrated_fields'), + ] + + operations = [ + migrations.RemoveField( + model_name='writeitinstancepopitinstancerecord', + name='popitapiinstance', + ), + ] diff --git a/instance/models.py b/instance/models.py index 874ec0ac..243a51b4 100644 --- a/instance/models.py +++ b/instance/models.py @@ -148,7 +148,6 @@ class WriteitInstancePopitInstanceRecord(models.Model): ("inprogress", _("In Progress")), ) writeitinstance = models.ForeignKey(WriteItInstance) - popitapiinstance = models.ForeignKey(ApiInstance) popolo_source = models.ForeignKey(PopoloSource) periodicity = models.CharField( max_length="2", From 774c87ed786509069848f798e4327bb68dc4dd0d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 14 Jul 2016 12:31:30 +0100 Subject: [PATCH 10/51] Add a note about how django-popit needs to be still installed :( --- writeit/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/writeit/settings.py b/writeit/settings.py index 276b81c3..4104b0e7 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -211,7 +211,11 @@ 'nuntium', 'djangoplugins', 'pagination', + + # Although django-popit is unused now, we need to keep it + # installed because the earlier migrations depend on its presence. 'popit', + 'popolo', 'popolo_sources', 'contactos', From dd65d9555a9efb3b3e1c710704525b6fa9f13ab5 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 14 Jul 2016 12:36:21 +0100 Subject: [PATCH 11/51] Remove an import from django-popit in global_test_case.py --- global_test_case.py | 1 - 1 file changed, 1 deletion(-) diff --git a/global_test_case.py b/global_test_case.py index 45fa44d1..7c88e34f 100644 --- a/global_test_case.py +++ b/global_test_case.py @@ -6,7 +6,6 @@ from tastypie.test import ResourceTestCase from django.conf import settings from django.contrib.sites.models import Site -from popit.tests import instance_helpers import os import subprocess import threading From 8162c9529bb966f5fb6813023a678d3b50d64b78 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 14 Jul 2016 12:32:59 +0100 Subject: [PATCH 12/51] Update the example_data.yaml fixture for django-popolo models --- example_data.yaml | 113 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 12 deletions(-) diff --git a/example_data.yaml b/example_data.yaml index dc88db61..65f4d691 100644 --- a/example_data.yaml +++ b/example_data.yaml @@ -60,20 +60,46 @@ - fields: {contact: 1, message: 4, site: 1} model: nuntium.outboundmessage pk: 5 -- fields: {url: 'http://popit.org/api/v1'} - model: popit.apiinstance +- fields: + url: 'http://popit.org/api/v1' + model: popolo_sources.popolosource pk: 1 -- fields: {url: 'http://popit.mysociety.org/api/v1/'} - model: popit.apiinstance +- fields: + url: 'http://popit.mysociety.org/api/v1/' + model: popolo_sources.popolosource + pk: 2 + +- model: popolo_sources.linktopopolosource + fields: + popolo_source: 1 + deleted_from_source: false + content_type: [popolo, person] + object_id: 1 + pk: 1 +- model: popolo_sources.linktopopolosource + fields: + popolo_source: 2 + deleted_from_source: false + content_type: [popolo, person] + object_id: 2 + pk: 2 +- model: popolo_sources.linktopopolosource + fields: + popolo_source: 2 + deleted_from_source: false + content_type: [popolo, person] + object_id: 3 pk: 2 -- fields: {api_instance: 1, name: Pedro, popit_url: 'http://popit.mysociety.org/api/v1/persons/1', popit_id: '52bc7asdasd34567'} - model: popit.person + + +- fields: {name: Pedro} + model: popolo.person pk: 1 -- fields: {api_instance: 2, name: Marcel, popit_url: 'http://popit.mysociety.org/api/v1/persons/2', popit_id: '52bc7asdasd34568'} - model: popit.person +- fields: {name: Marcel} + model: popolo.person pk: 2 -- fields: {api_instance: 2, name: Felipe, popit_url: 'http://popit.mysociety.org/api/v1/persons/3', popit_id: '52bc7asdasd34569'} - model: popit.person +- fields: {name: Felipe} + model: popolo.person pk: 3 - fields: {content: Public Answer,message: 2,person: 1} model: nuntium.answer @@ -82,10 +108,10 @@ model: nuntium.answer pk: 2 - fields: {person: 1, writeitinstance: 1} - model: instance.membership + model: instance.instancemembership pk: 1 - fields: {person: 1, writeitinstance: 2} - model: instance.membership + model: instance.instancemembership pk: 2 - fields: {label_name: Electronic Mail, name: e-mail} model: contactos.contacttype @@ -102,3 +128,66 @@ - fields: {contact_type: 1, person: 3, value: falvarez@ciudadanointeligente.cl, writeitinstance: 1} model: contactos.contact pk: 4 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34567 + object_id: 1 + scheme: popit_id + model: popolo.identifier + pk: 1 +- fields: + content_type: [popolo, person] + identifier: http://popit.mysociety.org/api/v1/persons/1 + object_id: 1 + scheme: popit_url + model: popolo.identifier + pk: 2 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34567 + object_id: 1 + scheme: popolo:person + model: popolo.identifier + pk: 3 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34568 + object_id: 2 + scheme: popit_id + model: popolo.identifier + pk: 4 +- fields: + content_type: [popolo, person] + identifier: http://popit.mysociety.org/api/v1/persons/2 + object_id: 2 + scheme: popit_url + model: popolo.identifier + pk: 5 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34568 + object_id: 2 + scheme: popolo:person + model: popolo.identifier + pk: 6 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34569 + object_id: 3 + scheme: popit_id + model: popolo.identifier + pk: 7 +- fields: + content_type: [popolo, person] + identifier: http://popit.mysociety.org/api/v1/persons/3 + object_id: 3 + scheme: popit_url + model: popolo.identifier + pk: 8 +- fields: + content_type: [popolo, person] + identifier: 52bc7asdasd34569 + object_id: 3 + scheme: popolo:person + model: popolo.identifier + pk: 9 From 5f97957f787b876973b1935663b54103dbfd9f95 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:45:55 +0100 Subject: [PATCH 13/51] Switch code from Membership to InstanceMembership --- instance/models.py | 7 ++++--- mailit/tests/save_raw_email_tests.py | 2 +- nuntium/models.py | 2 +- nuntium/tests/instance_config_tests.py | 4 ++-- nuntium/tests/message_form_tests.py | 4 ++-- nuntium/tests/popit_writeit_relation_tests.py | 18 +++++++++--------- nuntium/tests/writeitinstances_test.py | 4 ++-- .../tests/popit_instance_update_tests.py | 4 ++-- 8 files changed, 23 insertions(+), 22 deletions(-) diff --git a/instance/models.py b/instance/models.py index 243a51b4..a6291f86 100644 --- a/instance/models.py +++ b/instance/models.py @@ -29,7 +29,8 @@ class WriteItInstance(models.Model): owner = models.ForeignKey(User, related_name="writeitinstances") def add_person(self, person): - Membership.objects.get_or_create(writeitinstance=self, person=person) + InstanceMembership.objects.get_or_create( + writeitinstance=self, person=person) @property def persons_with_contacts(self): @@ -48,9 +49,9 @@ def relate_with_persons_from_popit_api_instance(self, popit_api_instance): persons = Person.objects.filter(api_instance=popit_api_instance) for person in persons: # There could be several memberships created. - memberships = Membership.objects.filter(writeitinstance=self, person=person) + memberships = InstanceMembership.objects.filter(writeitinstance=self, person=person) if memberships.count() == 0: - Membership.objects.create(writeitinstance=self, person=person) + InstanceMembership.objects.create(writeitinstance=self, person=person) if memberships.count() > 1: membership = memberships[0] memberships.exclude(id=membership.id).delete() diff --git a/mailit/tests/save_raw_email_tests.py b/mailit/tests/save_raw_email_tests.py index 30bb53e2..3964d05a 100644 --- a/mailit/tests/save_raw_email_tests.py +++ b/mailit/tests/save_raw_email_tests.py @@ -121,7 +121,7 @@ def save(self): def mock_request_to_api(self): person = self.outbound_message.message.people.get( - membership__writeitinstance=self.outbound_message.message.writeitinstance, + instancemembership__writeitinstance=self.outbound_message.message.writeitinstance, ) answer = Answer.objects.create( diff --git a/nuntium/models.py b/nuntium/models.py index 1e592dd1..e3a63818 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -323,7 +323,7 @@ class Answer(models.Model): created = models.DateTimeField(auto_now=True, null=True) def save(self, *args, **kwargs): - memberships = self.message.writeitinstance.membership_set.filter(person=self.person) + memberships = self.message.writeitinstance.instancemembership_set.filter(person=self.person) if memberships.count() == 0: raise AttributeError(_("This guy does not belong here")) super(Answer, self).save(*args, **kwargs) diff --git a/nuntium/tests/instance_config_tests.py b/nuntium/tests/instance_config_tests.py index 814c52a2..26c8daf9 100644 --- a/nuntium/tests/instance_config_tests.py +++ b/nuntium/tests/instance_config_tests.py @@ -1,6 +1,6 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from instance.models import Membership, WriteItInstance, WriteItInstanceConfig +from instance.models import InstanceMembership, WriteItInstance, WriteItInstanceConfig from nuntium.models import Message from popit.models import ApiInstance, Person from django.contrib.auth.models import User @@ -85,7 +85,7 @@ def setUp(self): name='instance 1', slug='instance-1', owner=self.owner) - Membership.objects.create(writeitinstance=self.writeitinstance, person=self.person1) + InstanceMembership.objects.create(writeitinstance=self.writeitinstance, person=self.person1) self.channel = MailChannel() self.contact = Contact.objects.create( diff --git a/nuntium/tests/message_form_tests.py b/nuntium/tests/message_form_tests.py index e6df53ec..d894a77b 100644 --- a/nuntium/tests/message_form_tests.py +++ b/nuntium/tests/message_form_tests.py @@ -2,7 +2,7 @@ from global_test_case import GlobalTestCase as TestCase from popit.models import Person from contactos.models import Contact -from instance.models import Membership, WriteItInstance +from instance.models import InstanceMembership, WriteItInstance from ..models import Message, Confirmation, OutboundMessage from ..forms import MessageCreateForm, PersonMultipleChoiceField @@ -186,7 +186,7 @@ def test_maximum_recipients(self): self.writeitinstance1.config.maximum_recipients = 1 self.writeitinstance1.config.save() person2 = Person.objects.get(id=2) - Membership.objects.create(writeitinstance=self.writeitinstance1, person=person2) + InstanceMembership.objects.create(writeitinstance=self.writeitinstance1, person=person2) data = { 'subject': u'Amor a la fiera', 'content': u'Todos sabemos que quieres mucho a la Fiera pero... es verdad?', diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 7a8733ed..77f4e1a4 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -1,7 +1,7 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import ( - Membership, WriteItInstance, WriteitInstancePopitInstanceRecord) + InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) from popit.models import ApiInstance from django.utils.unittest import skip from django.contrib.auth.models import User @@ -67,7 +67,7 @@ def test_it_does_not_try_to_replicate_the_memberships(self): writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) - amount_of_memberships = Membership.objects.filter(writeitinstance=writeitinstance).count() + amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() # There are only 2 self.assertEquals(amount_of_memberships, 2) @@ -80,22 +80,22 @@ def test_clean_memberships(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # there should be an amount of memberships writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) - amount_of_memberships = Membership.objects.filter(writeitinstance=writeitinstance).count() - previous_memberships = list(Membership.objects.filter(writeitinstance=writeitinstance)) + amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() + previous_memberships = list(InstanceMembership.objects.filter(writeitinstance=writeitinstance)) person = writeitinstance.persons.all()[0] # Creating a new one - Membership.objects.create(writeitinstance=writeitinstance, person=person) + InstanceMembership.objects.create(writeitinstance=writeitinstance, person=person) try: with popit_load_data(): writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) - except Membership.MultipleObjectsReturned, e: - self.fail("There are more than one Membership " + e) + except InstanceMembership.MultipleObjectsReturned, e: + self.fail("There are more than one InstanceMembership " + e) # It deletes the bad membership - new_amount_of_memberships = Membership.objects.filter(writeitinstance=writeitinstance).count() - later_memberships = list(Membership.objects.filter(writeitinstance=writeitinstance)) + new_amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() + later_memberships = list(InstanceMembership.objects.filter(writeitinstance=writeitinstance)) self.assertEquals(amount_of_memberships, new_amount_of_memberships) self.assertEquals(previous_memberships, later_memberships) diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index bbfba15f..2b404393 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -2,7 +2,7 @@ from urlparse import urlsplit, urlunsplit from global_test_case import GlobalTestCase as TestCase, popit_load_data from subdomains.utils import reverse -from instance.models import Membership, WriteItInstance +from instance.models import InstanceMembership, WriteItInstance from nuntium.models import Message, Confirmation from popit.models import ApiInstance, Person from django.utils.unittest import skip @@ -108,7 +108,7 @@ def test_get_non_existing_instance(self): def test_membership(self): writeitinstance = WriteItInstance.objects.create(name=u'instance 1', slug=u'instance-1', owner=self.owner) - Membership.objects.create(writeitinstance=writeitinstance, person=self.person1) + InstanceMembership.objects.create(writeitinstance=writeitinstance, person=self.person1) self.assertEquals(writeitinstance.persons.get(id=self.person1.id), self.person1) self.assertEquals(self.person1.writeit_instances.get(id=writeitinstance.id), writeitinstance) diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 7248a70e..a7ce66cc 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -1,7 +1,7 @@ from global_test_case import popit_load_data from subdomains.utils import reverse from instance.models import ( - Membership, WriteItInstance, WriteitInstancePopitInstanceRecord) + InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) from django.contrib.auth.models import User from django.forms import Form, URLField from django.conf import settings @@ -48,7 +48,7 @@ def test_update_creates_records_given_an_instance_2_persons(self): api_instance=w.persons.first().api_instance, name="Another Person but with the same api Instance", ) - Membership.objects.create(writeitinstance=w, person=another_person) + InstanceMembership.objects.create(writeitinstance=w, person=another_person) WPBackfillRecords.back_fill_popit_records(writeitinstance=w) records = WriteitInstancePopitInstanceRecord.objects.filter(writeitinstance=w) self.assertEquals(records.count(), 1) From e1830668389449085d07205f1745e05014b62b3a Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:48:18 +0100 Subject: [PATCH 14/51] Update the admin to use the new django-popolo models --- nuntium/admin.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/nuntium/admin.py b/nuntium/admin.py index b0a5eb02..26ae2795 100644 --- a/nuntium/admin.py +++ b/nuntium/admin.py @@ -5,16 +5,13 @@ ConfirmationTemplate from instance.models import WriteItInstance -from popit.models import ApiInstance, Person from mailit.models import MailItTemplate +from popolo.models import Person +from popolo_sources.models import PopoloSource from django_object_actions import DjangoObjectActions from nuntium.forms import WriteItInstanceCreateFormPopitUrl -class PersonInline(admin.TabularInline): - model = Person - - class MembershipInline(admin.TabularInline): model = WriteItInstance.persons.through @@ -106,14 +103,9 @@ class MessageRecordAdmin(admin.ModelAdmin): admin.site.register(MessageRecord, MessageRecordAdmin) -class ApiInstanceAdmin(admin.ModelAdmin): - pass -admin.site.register(ApiInstance, ApiInstanceAdmin) - - -class PersonAdmin(admin.ModelAdmin): +class PopoloSourceAdmin(admin.ModelAdmin): pass -admin.site.register(Person, PersonAdmin) +admin.site.register(PopoloSource, PopoloSourceAdmin) class NewAnswerNotificationTemplateAdmin(admin.ModelAdmin): From 755e5a82e576a99e304d4ac7f8bc6766aa49dfb6 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 15 Jul 2016 14:49:30 +0100 Subject: [PATCH 15/51] Fix naive datettime warnings on loaddata --- example_data.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_data.yaml b/example_data.yaml index 65f4d691..9b9f31c8 100644 --- a/example_data.yaml +++ b/example_data.yaml @@ -2,14 +2,14 @@ model: sites.site pk: 1 - fields: - date_joined: 2013-04-26 21:09:22.224616+00:00 + date_joined: '2013-04-26T21:09:22Z' email: admin@admines.cl first_name: 'Administrador del system' groups: [] is_active: true is_staff: true is_superuser: true - last_login: 2013-06-10 16:06:46.989439+00:00 + last_login: '2013-06-10T16:06:46Z' last_name: '' password: pbkdf2_sha256$10000$LoeTDV2yMCWT$fgkQ++r4O7wurfFSAtHT1/a9WTjb32cHq63dFgXU+rI= user_permissions: [] From 596690a91e48d940407e950336c45a7f8cf60fad Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:49:36 +0100 Subject: [PATCH 16/51] Switch Person imports from popit-django to django-popolo --- contactos/tests/contacts_tests.py | 2 +- contactos/views.py | 2 +- instance/models.py | 2 +- mailit/tests/plugin_test.py | 2 +- nuntium/api.py | 2 +- nuntium/forms.py | 2 +- nuntium/models.py | 3 +-- nuntium/tests/all_messages_with_moderation_test.py | 2 +- nuntium/tests/answer_attachments_tests.py | 2 +- nuntium/tests/answers_test.py | 2 +- nuntium/tests/api/answer_resource_test.py | 2 +- nuntium/tests/api/instance_resource_test.py | 2 +- nuntium/tests/api/message_resource_test.py | 2 +- nuntium/tests/api/person_resource_test.py | 2 +- nuntium/tests/confirmation_template_test.py | 2 +- nuntium/tests/confirmation_test.py | 2 +- nuntium/tests/instance_config_tests.py | 3 ++- nuntium/tests/message_creation_tests.py | 2 +- nuntium/tests/message_detail_view_test.py | 2 +- nuntium/tests/message_form_tests.py | 2 +- nuntium/tests/messages_per_person_view_test.py | 2 +- nuntium/tests/messages_search_test.py | 2 +- nuntium/tests/messages_test.py | 3 ++- nuntium/tests/moderation_messages_test.py | 2 +- nuntium/tests/outbound_message_plugin_record_test.py | 2 +- nuntium/tests/outbound_message_test.py | 3 ++- nuntium/tests/public_messages_test.py | 2 +- nuntium/tests/rate_limiter_tests.py | 2 +- nuntium/tests/record_system_test.py | 2 +- nuntium/tests/rtl_messages_test.py | 2 +- nuntium/tests/subscribers_test.py | 2 +- nuntium/tests/tasks_test.py | 2 +- nuntium/user_section/forms.py | 2 +- nuntium/user_section/tests/manually_create_answers_tests.py | 2 +- nuntium/user_section/tests/popit_instance_update_tests.py | 3 ++- nuntium/user_section/tests/template_tags_tests.py | 2 +- nuntium/user_section/tests/user_section_views_tests.py | 2 +- nuntium/views.py | 2 +- 38 files changed, 42 insertions(+), 39 deletions(-) diff --git a/contactos/tests/contacts_tests.py b/contactos/tests/contacts_tests.py index 7f36beb6..39691091 100644 --- a/contactos/tests/contacts_tests.py +++ b/contactos/tests/contacts_tests.py @@ -1,7 +1,7 @@ from global_test_case import GlobalTestCase as TestCase from nuntium.user_section.tests.user_section_views_tests import UserSectionTestCase from contactos.models import ContactType, Contact -from popit.models import Person +from popolo.models import Person from django.contrib.auth.models import User from django.utils.translation import ugettext as _ from django.core import mail diff --git a/contactos/views.py b/contactos/views.py index 846defaf..01271281 100644 --- a/contactos/views.py +++ b/contactos/views.py @@ -7,7 +7,7 @@ from django.utils.decorators import method_decorator from subdomains.utils import reverse from instance.models import WriteItInstance -from popit.models import Person +from popolo.models import Person from django.http import Http404 diff --git a/instance/models.py b/instance/models.py index a6291f86..bb410ed1 100644 --- a/instance/models.py +++ b/instance/models.py @@ -10,7 +10,7 @@ from annoying.fields import AutoOneToOneField from autoslug import AutoSlugField from nuntium.popit_api_instance import PopitApiInstance -from popit.models import Person, ApiInstance +from popit.models import ApiInstance from popolo.models import Person as PopoloPerson from popolo_sources.models import PopoloSource from requests.exceptions import ConnectionError diff --git a/mailit/tests/plugin_test.py b/mailit/tests/plugin_test.py index 89eff4a9..a6b0546a 100644 --- a/mailit/tests/plugin_test.py +++ b/mailit/tests/plugin_test.py @@ -17,7 +17,7 @@ from nuntium.plugins import OutputPlugin from nuntium.user_section.tests.user_section_views_tests import UserSectionTestCase -from popit.models import Person +from popolo.models import Person from global_test_case import GlobalTestCase as TestCase diff --git a/nuntium/api.py b/nuntium/api.py index f04f8624..d3f84679 100644 --- a/nuntium/api.py +++ b/nuntium/api.py @@ -10,7 +10,7 @@ from tastypie import fields from tastypie.exceptions import ImmediateHttpResponse from tastypie import http -from popit.models import Person +from popolo.models import Person from contactos.models import Contact from tastypie.paginator import Paginator from django.http import Http404, HttpResponseBadRequest diff --git a/nuntium/forms.py b/nuntium/forms.py index e1ef9490..0017c2e9 100644 --- a/nuntium/forms.py +++ b/nuntium/forms.py @@ -6,9 +6,9 @@ from .models import Message, Confirmation from contactos.models import Contact +from popolo.models import Person from django.forms import ValidationError from django.utils.translation import ugettext as _, ungettext -from popit.models import Person from haystack.forms import SearchForm from django.utils.html import format_html from django.utils.encoding import force_text diff --git a/nuntium/models.py b/nuntium/models.py index e3a63818..cc1e28e7 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -5,7 +5,6 @@ from django.db import models from django.utils.translation import override, ugettext_lazy as _ from django.core.exceptions import ValidationError, ObjectDoesNotExist -from popit.models import Person from popolo.models import Person as PopoloPerson from contactos.models import Contact from .plugins import OutputPlugin @@ -169,7 +168,7 @@ def recently_confirmated(self): @property def people(self): - people = Person.objects.filter( + people = PopoloPerson.objects.filter( Q(contact__outboundmessage__message=self) | Q(nocontactom__message=self) ).distinct() diff --git a/nuntium/tests/all_messages_with_moderation_test.py b/nuntium/tests/all_messages_with_moderation_test.py index d87c21e5..285c916a 100644 --- a/nuntium/tests/all_messages_with_moderation_test.py +++ b/nuntium/tests/all_messages_with_moderation_test.py @@ -2,7 +2,7 @@ from global_test_case import GlobalTestCase as TestCase from instance.models import WriteItInstance from ..models import Message, Moderation -from popit.models import Person +from popolo.models import Person from django.core import mail diff --git a/nuntium/tests/answer_attachments_tests.py b/nuntium/tests/answer_attachments_tests.py index d73f310e..7cf1f42d 100644 --- a/nuntium/tests/answer_attachments_tests.py +++ b/nuntium/tests/answer_attachments_tests.py @@ -1,6 +1,6 @@ from global_test_case import GlobalTestCase as TestCase from nuntium.models import Message, Answer, AnswerAttachment -from popit.models import Person +from popolo.models import Person from django.core.files import File from subdomains.utils import reverse diff --git a/nuntium/tests/answers_test.py b/nuntium/tests/answers_test.py index ac832571..e0ad51eb 100644 --- a/nuntium/tests/answers_test.py +++ b/nuntium/tests/answers_test.py @@ -1,6 +1,6 @@ from global_test_case import GlobalTestCase as TestCase from ..models import Message, Answer -from popit.models import Person +from popolo.models import Person from django.utils.translation import ugettext as _ diff --git a/nuntium/tests/api/answer_resource_test.py b/nuntium/tests/api/answer_resource_test.py index 81fb642c..b75a8218 100644 --- a/nuntium/tests/api/answer_resource_test.py +++ b/nuntium/tests/api/answer_resource_test.py @@ -3,7 +3,7 @@ from instance.models import WriteItInstance from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popit.models import Person +from popolo.models import Person from ...api import AnswerResource from django.http import HttpRequest from ...models import Answer, Message diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index 872cf2bc..de6d8667 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -4,7 +4,7 @@ from ...models import Message, Confirmation from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popit.models import Person +from popolo.models import Person from global_test_case import popit_load_data from django.conf import settings import re diff --git a/nuntium/tests/api/message_resource_test.py b/nuntium/tests/api/message_resource_test.py index cbfdc5e3..01bf31c1 100644 --- a/nuntium/tests/api/message_resource_test.py +++ b/nuntium/tests/api/message_resource_test.py @@ -4,7 +4,7 @@ from ...models import Message, Confirmation from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popit.models import Person +from popolo.models import Person from django.utils.encoding import force_text diff --git a/nuntium/tests/api/person_resource_test.py b/nuntium/tests/api/person_resource_test.py index 6837304a..a1d29b07 100644 --- a/nuntium/tests/api/person_resource_test.py +++ b/nuntium/tests/api/person_resource_test.py @@ -3,7 +3,7 @@ from instance.models import WriteItInstance from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popit.models import Person +from popolo.models import Person class PersonResourceTestCase(ResourceTestCase): diff --git a/nuntium/tests/confirmation_template_test.py b/nuntium/tests/confirmation_template_test.py index ecdc248f..75318fb7 100644 --- a/nuntium/tests/confirmation_template_test.py +++ b/nuntium/tests/confirmation_template_test.py @@ -11,7 +11,7 @@ from django.test.utils import override_settings from contactos.models import Contact, ContactType -from popit.models import Person +from popolo.models import Person import codecs import os diff --git a/nuntium/tests/confirmation_test.py b/nuntium/tests/confirmation_test.py index 0859f5a0..cc6060d2 100644 --- a/nuntium/tests/confirmation_test.py +++ b/nuntium/tests/confirmation_test.py @@ -2,7 +2,7 @@ from instance.models import WriteItInstance from ..models import Confirmation, OutboundMessage from ..models import Message -from popit.models import Person +from popolo.models import Person from contactos.models import Contact from datetime import datetime from django.core import mail diff --git a/nuntium/tests/instance_config_tests.py b/nuntium/tests/instance_config_tests.py index 26c8daf9..4f936bbd 100644 --- a/nuntium/tests/instance_config_tests.py +++ b/nuntium/tests/instance_config_tests.py @@ -2,7 +2,8 @@ from global_test_case import GlobalTestCase as TestCase from instance.models import InstanceMembership, WriteItInstance, WriteItInstanceConfig from nuntium.models import Message -from popit.models import ApiInstance, Person +from popit.models import ApiInstance +from popolo.models import Person from django.contrib.auth.models import User from mailit import MailChannel from contactos.models import Contact diff --git a/nuntium/tests/message_creation_tests.py b/nuntium/tests/message_creation_tests.py index 7b90a198..616e30a8 100644 --- a/nuntium/tests/message_creation_tests.py +++ b/nuntium/tests/message_creation_tests.py @@ -3,7 +3,7 @@ from instance.models import WriteItInstance from nuntium.models import Message from subdomains.utils import reverse -from popit.models import Person +from popolo.models import Person from nuntium.forms import WhoForm, DraftForm diff --git a/nuntium/tests/message_detail_view_test.py b/nuntium/tests/message_detail_view_test.py index 5351712e..a625e4eb 100644 --- a/nuntium/tests/message_detail_view_test.py +++ b/nuntium/tests/message_detail_view_test.py @@ -2,7 +2,7 @@ from global_test_case import GlobalTestCase as TestCase from instance.models import WriteItInstance from ..models import Message, Confirmation -from popit.models import Person +from popolo.models import Person import datetime from nuntium.views import MessageThreadView diff --git a/nuntium/tests/message_form_tests.py b/nuntium/tests/message_form_tests.py index d894a77b..f067ee8f 100644 --- a/nuntium/tests/message_form_tests.py +++ b/nuntium/tests/message_form_tests.py @@ -1,6 +1,6 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from popit.models import Person +from popolo.models import Person from contactos.models import Contact from instance.models import InstanceMembership, WriteItInstance from ..models import Message, Confirmation, OutboundMessage diff --git a/nuntium/tests/messages_per_person_view_test.py b/nuntium/tests/messages_per_person_view_test.py index e1ed6494..bed35dbb 100644 --- a/nuntium/tests/messages_per_person_view_test.py +++ b/nuntium/tests/messages_per_person_view_test.py @@ -3,7 +3,7 @@ from subdomains.utils import reverse from instance.models import WriteItInstance from ..models import Message -from popit.models import Person +from popolo.models import Person class MessagesPerPersonViewTestCase(TestCase): diff --git a/nuntium/tests/messages_search_test.py b/nuntium/tests/messages_search_test.py index 04dc39ed..a4e08ad2 100644 --- a/nuntium/tests/messages_search_test.py +++ b/nuntium/tests/messages_search_test.py @@ -12,7 +12,7 @@ from instance.models import WriteItInstance from ..views import MessageSearchView, PerInstanceSearchView from haystack.views import SearchView -from popit.models import Person +from popolo.models import Person import urllib import urlparse diff --git a/nuntium/tests/messages_test.py b/nuntium/tests/messages_test.py index 9fa9ad3c..2c43ef63 100644 --- a/nuntium/tests/messages_test.py +++ b/nuntium/tests/messages_test.py @@ -7,7 +7,8 @@ from contactos.models import Contact from instance.models import WriteItInstance from ..models import Message, OutboundMessage, NoContactOM -from popit.models import Person, ApiInstance +from popit.models import ApiInstance +from popolo.models import Person from subdomains.utils import reverse from django.contrib.auth.models import User from django.template.defaultfilters import slugify diff --git a/nuntium/tests/moderation_messages_test.py b/nuntium/tests/moderation_messages_test.py index c4146797..92282079 100644 --- a/nuntium/tests/moderation_messages_test.py +++ b/nuntium/tests/moderation_messages_test.py @@ -2,7 +2,7 @@ from global_test_case import GlobalTestCase as TestCase from ..models import Message, WriteItInstance, \ Moderation, Confirmation, OutboundMessage -from popit.models import Person +from popolo.models import Person from django.core import mail from subdomains.utils import reverse import datetime diff --git a/nuntium/tests/outbound_message_plugin_record_test.py b/nuntium/tests/outbound_message_plugin_record_test.py index 92ae9623..47fd4793 100644 --- a/nuntium/tests/outbound_message_plugin_record_test.py +++ b/nuntium/tests/outbound_message_plugin_record_test.py @@ -5,7 +5,7 @@ from contactos.models import Contact # from djangoplugins.models import Plugin from django.contrib.auth.models import User -from popit.models import Person +from popolo.models import Person ''' This testcase is intented to test the OutboundMessageRecord model and it's creation when sending an outbound_message, the calculation of diff --git a/nuntium/tests/outbound_message_test.py b/nuntium/tests/outbound_message_test.py index be37a887..17e32bec 100644 --- a/nuntium/tests/outbound_message_test.py +++ b/nuntium/tests/outbound_message_test.py @@ -10,7 +10,8 @@ NoContactOM, AbstractOutboundMessage from django.contrib.sites.models import Site -from popit.models import Person, ApiInstance +from popit.models import ApiInstance +from popolo.models import Person from mock import patch from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType diff --git a/nuntium/tests/public_messages_test.py b/nuntium/tests/public_messages_test.py index 6393542f..a14cd8ca 100644 --- a/nuntium/tests/public_messages_test.py +++ b/nuntium/tests/public_messages_test.py @@ -3,7 +3,7 @@ from global_test_case import GlobalTestCase as TestCase from instance.models import WriteItInstance from ..models import Message, Confirmation -from popit.models import Person +from popolo.models import Person from django.contrib.auth.models import User from tastypie.test import ResourceTestCase, TestApiClient diff --git a/nuntium/tests/rate_limiter_tests.py b/nuntium/tests/rate_limiter_tests.py index cc6c2091..a2d952a1 100644 --- a/nuntium/tests/rate_limiter_tests.py +++ b/nuntium/tests/rate_limiter_tests.py @@ -5,7 +5,7 @@ from ..models import RateLimiter, Message from datetime import date from django.core.exceptions import ValidationError -from popit.models import Person +from popolo.models import Person from django.utils.translation import ugettext as _ diff --git a/nuntium/tests/record_system_test.py b/nuntium/tests/record_system_test.py index 058058ef..064b0720 100644 --- a/nuntium/tests/record_system_test.py +++ b/nuntium/tests/record_system_test.py @@ -2,7 +2,7 @@ from instance.models import WriteItInstance from ..models import Message, OutboundMessage, MessageRecord from django.contrib.contenttypes.models import ContentType -from popit.models import Person +from popolo.models import Person from django.utils.translation import ugettext as _ import datetime diff --git a/nuntium/tests/rtl_messages_test.py b/nuntium/tests/rtl_messages_test.py index 98d3bf87..e87fd92e 100644 --- a/nuntium/tests/rtl_messages_test.py +++ b/nuntium/tests/rtl_messages_test.py @@ -2,7 +2,7 @@ from global_test_case import GlobalTestCase as TestCase from instance.models import WriteItInstance from ..models import Message -from popit.models import Person +from popolo.models import Person class RTLTextInMessages(TestCase): diff --git a/nuntium/tests/subscribers_test.py b/nuntium/tests/subscribers_test.py index 94bda165..16dff878 100644 --- a/nuntium/tests/subscribers_test.py +++ b/nuntium/tests/subscribers_test.py @@ -2,7 +2,7 @@ from instance.models import WriteItInstance from ..models import Subscriber, Message, \ Confirmation, Answer, NewAnswerNotificationTemplate -from popit.models import Person +from popolo.models import Person from django.contrib.auth.models import User from django.core import mail from django.conf import settings diff --git a/nuntium/tests/tasks_test.py b/nuntium/tests/tasks_test.py index d1af3356..7cc9776e 100644 --- a/nuntium/tests/tasks_test.py +++ b/nuntium/tests/tasks_test.py @@ -4,7 +4,7 @@ from nuntium.models import OutboundMessage from ..tasks import send_mails_task from mock import patch -from popit.models import Person +from popolo.models import Person from nuntium.popit_api_instance import PopitApiInstance from django.contrib.auth.models import User from django.conf import settings diff --git a/nuntium/user_section/forms.py b/nuntium/user_section/forms.py index ac7badc4..11f915f8 100644 --- a/nuntium/user_section/forms.py +++ b/nuntium/user_section/forms.py @@ -20,7 +20,7 @@ from django.utils.translation import ugettext as _ -from popit.models import Person +from popolo.models import Person from instance.models import ( WriteItInstance, diff --git a/nuntium/user_section/tests/manually_create_answers_tests.py b/nuntium/user_section/tests/manually_create_answers_tests.py index d7fd27e3..622e923f 100644 --- a/nuntium/user_section/tests/manually_create_answers_tests.py +++ b/nuntium/user_section/tests/manually_create_answers_tests.py @@ -3,7 +3,7 @@ from ...models import Message, Answer from django.contrib.auth.models import User from ..forms import AnswerForm -from popit.models import Person +from popolo.models import Person from user_section_views_tests import UserSectionTestCase diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index a7ce66cc..1abcfdb2 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -6,7 +6,8 @@ from django.forms import Form, URLField from django.conf import settings from django.core.management import call_command -from popit.models import Person, ApiInstance +from popit.models import ApiInstance +from popolo.models import Person from django.utils.unittest import skip from user_section_views_tests import UserSectionTestCase from django.utils.translation import ugettext as _ diff --git a/nuntium/user_section/tests/template_tags_tests.py b/nuntium/user_section/tests/template_tags_tests.py index 5f597e55..1d77401a 100644 --- a/nuntium/user_section/tests/template_tags_tests.py +++ b/nuntium/user_section/tests/template_tags_tests.py @@ -1,7 +1,7 @@ from django.template import Context, Template from global_test_case import GlobalTestCase as TestCase from contactos.models import Contact -from popit.models import Person +from popolo.models import Person from subdomains.utils import reverse from django.template import TemplateSyntaxError diff --git a/nuntium/user_section/tests/user_section_views_tests.py b/nuntium/user_section/tests/user_section_views_tests.py index dbe8f861..48fae046 100644 --- a/nuntium/user_section/tests/user_section_views_tests.py +++ b/nuntium/user_section/tests/user_section_views_tests.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.conf import settings from django.test.client import RequestFactory -from popit.models import Person +from popolo.models import Person from mailit.forms import MailitTemplateForm from global_test_case import GlobalTestCase as TestCase, popit_load_data diff --git a/nuntium/views.py b/nuntium/views.py index 174aa211..3700640c 100644 --- a/nuntium/views.py +++ b/nuntium/views.py @@ -8,7 +8,7 @@ from haystack.views import SearchView from itertools import chain -from popit.models import Person +from popolo.models import Person from django.db.models import Q from instance.models import WriteItInstance from .models import Confirmation, Message, Moderation From 0c76a00a27faf27f00d44a41b122dc8998546e26 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 11:25:59 +0100 Subject: [PATCH 17/51] Remove imports of the popit-django ApiInstance --- nuntium/tests/messages_test.py | 1 - nuntium/tests/outbound_message_test.py | 1 - nuntium/tests/popit_writeit_relation_tests.py | 1 - nuntium/tests/writeitinstances_test.py | 2 +- nuntium/user_section/tests/popit_instance_update_tests.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/nuntium/tests/messages_test.py b/nuntium/tests/messages_test.py index 2c43ef63..e396fd91 100644 --- a/nuntium/tests/messages_test.py +++ b/nuntium/tests/messages_test.py @@ -7,7 +7,6 @@ from contactos.models import Contact from instance.models import WriteItInstance from ..models import Message, OutboundMessage, NoContactOM -from popit.models import ApiInstance from popolo.models import Person from subdomains.utils import reverse from django.contrib.auth.models import User diff --git a/nuntium/tests/outbound_message_test.py b/nuntium/tests/outbound_message_test.py index 17e32bec..41325a35 100644 --- a/nuntium/tests/outbound_message_test.py +++ b/nuntium/tests/outbound_message_test.py @@ -10,7 +10,6 @@ NoContactOM, AbstractOutboundMessage from django.contrib.sites.models import Site -from popit.models import ApiInstance from popolo.models import Person from mock import patch from django.contrib.auth.models import User diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 77f4e1a4..0cbce10a 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -2,7 +2,6 @@ from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import ( InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) -from popit.models import ApiInstance from django.utils.unittest import skip from django.contrib.auth.models import User from django.conf import settings diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index 2b404393..fd8a90b4 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -4,7 +4,7 @@ from subdomains.utils import reverse from instance.models import InstanceMembership, WriteItInstance from nuntium.models import Message, Confirmation -from popit.models import ApiInstance, Person +from popolo.models import Person from django.utils.unittest import skip from django.contrib.auth.models import User from django.utils.translation import activate diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 1abcfdb2..4a2871a2 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -6,7 +6,6 @@ from django.forms import Form, URLField from django.conf import settings from django.core.management import call_command -from popit.models import ApiInstance from popolo.models import Person from django.utils.unittest import skip from user_section_views_tests import UserSectionTestCase From cdd85798ef9be046fa19b14cdeeca103adf6da2d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:38:14 +0100 Subject: [PATCH 18/51] Add a proxy model PopoloPerson with helpful properties --- ...d_popoloperson_proxy_and_change_related.py | 36 +++++++++ instance/models.py | 77 ++++++++++++++++++- 2 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 instance/migrations/0010_add_popoloperson_proxy_and_change_related.py diff --git a/instance/migrations/0010_add_popoloperson_proxy_and_change_related.py b/instance/migrations/0010_add_popoloperson_proxy_and_change_related.py new file mode 100644 index 00000000..7bbeb352 --- /dev/null +++ b/instance/migrations/0010_add_popoloperson_proxy_and_change_related.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('popolo', '0002_update_models_from_upstream'), + ('instance', '0008_remove_writeitinstancepopitinstancerecord_popitapiinstance'), + ] + + operations = [ + migrations.CreateModel( + name='PopoloPerson', + fields=[ + ], + options={ + 'proxy': True, + }, + bases=('popolo.person',), + ), + migrations.AlterField( + model_name='instancemembership', + name='person', + field=models.ForeignKey(to='instance.PopoloPerson'), + preserve_default=True, + ), + migrations.AlterField( + model_name='writeitinstance', + name='persons', + field=models.ManyToManyField(related_name='writeit_instances', through='instance.InstanceMembership', to='instance.PopoloPerson'), + preserve_default=True, + ), + ] diff --git a/instance/models.py b/instance/models.py index bb410ed1..a54acd88 100644 --- a/instance/models.py +++ b/instance/models.py @@ -2,6 +2,8 @@ from django.conf import settings from django.contrib.auth.models import User +from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.models import ContentType from django.core import mail from django.db import models from django.db.models.signals import post_save @@ -10,13 +12,84 @@ from annoying.fields import AutoOneToOneField from autoslug import AutoSlugField from nuntium.popit_api_instance import PopitApiInstance -from popit.models import ApiInstance -from popolo.models import Person as PopoloPerson from popolo_sources.models import PopoloSource +from popolo.models import Person from requests.exceptions import ConnectionError from subdomains.utils import reverse +class PopoloPerson(Person): + class Meta: + proxy = True + + objects = PopoloPersonQuerySet.as_manager() + + links_to_popolo_sources = GenericRelation( + LinkToPopoloSource, + related_query_name='people') + + @property + def popolo_source(self): + ct = ContentType.objects.get_for_model(Person) + link = LinkToPopoloSource.objects.get( + content_type=ct, object_id=self.id) + return link.popolo_source + + @property + def popolo_source_url(self): + return self.popolo_source.url + + # Note that these methods use the slightly odd implementation + # of iterating over the relation rather than using .filter or .get + # because if they're preloaded with prefetch_related those methods + # will incur an extra query - .all will not. + + @property + def id_in_popolo_source(self): + for i in self.identifiers.all(): + if i.scheme == 'popolo:person': + return i.identifier + + @property + def old_popit_url(self): + # We shouldn't be relying on this any more, but are still + # passing it in webhook payloads, so this property gives it an easy + for i in self.identifiers.all(): + if i.scheme == 'popit_url': + return i.identifier + + @property + def old_popit_id(self): + # We shouldn't be relying on this any more either, but it's + # used in tests and checks that old behaviour still works. + for i in self.identifiers.all(): + if i.scheme == 'popit_id': + return i.identifier + + def uri_for_api(self): + """Return the URL the tastypie API uses to refer to this person""" + + old_url = None + new_url = None + for i in self.identifiers.all(): + # If the old identifier exists, use this so that the IDs + # used in the API don't change. + if i.scheme == 'popit_url': + assert old_url is None + old_url = i.identifier + # For more recently created Person objects, popolo_uri is + # the best single identifier to use in the API: + elif i.scheme == 'popolo_uri': + assert new_url is None + new_url = i.identifier + if old_url: + return old_url + if new_url: + return new_url + msg = "Could find no global identifier for PopoloPerson with ID {0}" + raise PopoloIdentifier.DoesNotExist(msg.format(self.pk)) + + class WriteItInstance(models.Model): """WriteItInstance: Entity that groups messages and people for usability purposes. E.g. 'Candidates running for president'""" From df6d54340b65f4393e49ff44be3bfdb1305c355a Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 15 Jul 2016 18:01:58 +0100 Subject: [PATCH 19/51] Update the webhook payload and corresponding tests --- example_data.yaml | 12 ++++++------ nuntium/models.py | 6 ++++-- nuntium/tests/webhooks_tests.py | 8 ++++++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/example_data.yaml b/example_data.yaml index 9b9f31c8..8ff1d551 100644 --- a/example_data.yaml +++ b/example_data.yaml @@ -101,12 +101,6 @@ - fields: {name: Felipe} model: popolo.person pk: 3 -- fields: {content: Public Answer,message: 2,person: 1} - model: nuntium.answer - pk: 1 -- fields: {content: Private Answer,message: 3,person: 1} - model: nuntium.answer - pk: 2 - fields: {person: 1, writeitinstance: 1} model: instance.instancemembership pk: 1 @@ -191,3 +185,9 @@ scheme: popolo:person model: popolo.identifier pk: 9 +- fields: {content: Public Answer,message: 2,person: 1} + model: nuntium.answer + pk: 1 +- fields: {content: Private Answer,message: 3,person: 1} + model: nuntium.answer + pk: 2 diff --git a/nuntium/models.py b/nuntium/models.py index cc1e28e7..4952a5b7 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -391,8 +391,10 @@ def send_new_answer_payload(sender, instance, created, **kwargs): 'message_id': '/api/v1/message/{0}/'.format(answer.message.id), 'content': answer.content, 'person': answer.person.name, - 'person_id': answer.person.popit_url, - } + 'person_id': answer.person.uri_for_api(), + 'person_id_in_popolo_source': answer.person.id_in_popolo_source, + 'person_popolo_source_url': answer.person.popolo_source_url, + } for webhook in writeitinstance.answer_webhooks.all(): requests.post(webhook.url, data=payload) diff --git a/nuntium/tests/webhooks_tests.py b/nuntium/tests/webhooks_tests.py index 64c2431f..bb14c217 100644 --- a/nuntium/tests/webhooks_tests.py +++ b/nuntium/tests/webhooks_tests.py @@ -51,7 +51,9 @@ def test_when_a_new_answer_is_created_then_it_post_to_the_url(self): 'message_id': '/api/v1/message/{0}/'.format(message.id), 'content': 'holiwi', 'person': pedro.name, - 'person_id': pedro.popit_url, + 'person_id_in_popolo_source': pedro.id_in_popolo_source, + 'person_popolo_source_url': pedro.popolo_source_url, + 'person_id': pedro.uri_for_api(), } with patch('requests.post') as post: post.return_value = PostMock() @@ -72,7 +74,9 @@ def test_it_does_not_send_the_payload_twice(self): 'message_id': '/api/v1/message/{0}/'.format(message.id), 'content': 'holiwi', 'person': pedro.name, - 'person_id': pedro.popit_url, + 'person_id_in_popolo_source': pedro.id_in_popolo_source, + 'person_popolo_source_url': pedro.popolo_source_url, + 'person_id': pedro.uri_for_api(), } with patch('requests.post') as post: post.return_value = PostMock() From b1004098601ee26433ce3413c43a60b269b148ec Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:45:19 +0100 Subject: [PATCH 20/51] Remove the old PopIt proxy models --- instance/models.py | 1 - ...0006_auto_remove_old_popit_proxy_models.py | 22 ++++ nuntium/popit_api_instance.py | 111 ------------------ nuntium/tasks.py | 1 - nuntium/tests/api/instance_resource_test.py | 1 - nuntium/tests/popit_api_instance_tests.py | 1 - nuntium/tests/popit_writeit_relation_tests.py | 1 - nuntium/tests/tasks_test.py | 1 - nuntium/tests/writeitinstances_test.py | 1 - .../tests/popit_instance_update_tests.py | 1 - .../tests/user_section_views_tests.py | 1 - nuntium/user_section/views.py | 1 - 12 files changed, 22 insertions(+), 121 deletions(-) create mode 100644 nuntium/migrations/0006_auto_remove_old_popit_proxy_models.py delete mode 100644 nuntium/popit_api_instance.py diff --git a/instance/models.py b/instance/models.py index a54acd88..1f4f5dcc 100644 --- a/instance/models.py +++ b/instance/models.py @@ -11,7 +11,6 @@ from annoying.fields import AutoOneToOneField from autoslug import AutoSlugField -from nuntium.popit_api_instance import PopitApiInstance from popolo_sources.models import PopoloSource from popolo.models import Person from requests.exceptions import ConnectionError diff --git a/nuntium/migrations/0006_auto_remove_old_popit_proxy_models.py b/nuntium/migrations/0006_auto_remove_old_popit_proxy_models.py new file mode 100644 index 00000000..0b1ad817 --- /dev/null +++ b/nuntium/migrations/0006_auto_remove_old_popit_proxy_models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import datetime +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('nuntium', '0005_rename_migrated_fields'), + ] + + operations = [ + migrations.DeleteModel( + name='PopitApiInstance', + ), + migrations.DeleteModel( + name='PopitPerson', + ), + ] diff --git a/nuntium/popit_api_instance.py b/nuntium/popit_api_instance.py deleted file mode 100644 index 8b6cbc63..00000000 --- a/nuntium/popit_api_instance.py +++ /dev/null @@ -1,111 +0,0 @@ -from popit.models import ApiInstance, Person, get_paginated_generator -from contactos.models import Contact -from mailit import MailChannel -from datetime import datetime -import requests - - -def get_date_or_none(membership_doc, key, format="%Y-%m-%d"): - date = membership_doc.get(key, "") - if not date: - return None - try: - return datetime.strptime(date, format) - except TypeError: - return None - - -def _is_current_membership(start_date, end_date): - today = datetime.today() - is_valid = ((start_date is None) or start_date <= today) and\ - ((end_date is None) or end_date >= today) - return is_valid - - -def is_current_membership(membership_doc, start_date_key='start_date', end_date_key='end_date'): - start_date = get_date_or_none(membership_doc, start_date_key) - end_date = get_date_or_none(membership_doc, end_date_key) - return _is_current_membership(start_date, end_date) - - -class PopitPerson(Person): - class Meta: - proxy = True - - @classmethod - def fetch_all_from_api(cls, instance, writeitinstance): - """ - Get all the documents from the API and save them locally. - """ - cls = Person - - data = requests.get(instance.url).json() - if 'persons' in data: - # Direct Popolo data, list of persons with name/email/image/summary/inline memberships - collection_url = instance.url - people = data['persons'] - for p in people: - # XXX IDs in transformed EP have person/ at the front? - p['id'] = 'person/%s' % p['id'] - p.setdefault('memberships', []).extend( - [m for m in data['memberships'] if m['person_id'] == p['id']]) - else: - api_client = instance.api_client(cls.api_collection_name) - - # This is hacky, but I can't see a documented way to get to the url. - # Liable to change if slumber changes their internals. - collection_url = api_client._store['base_url'] - people = get_paginated_generator(api_client) - - for doc in people: - - # Add url to the doc - url = collection_url + '/' + doc['id'] - doc['popit_url'] = url - - obj = cls.update_from_api_results(instance=instance, doc=doc) - PopitPerson.create_contact(obj, doc, writeitinstance) - - @classmethod - def determine_if_person_is_current(cls, doc): - enable = any([is_current_membership(membership) for membership in doc['memberships']]) - return enable - - @classmethod - def create_contact(cls, obj, doc, writeitinstance): - contact_type = MailChannel().get_contact_type() - created_emails = [] - enable_contacts = cls.determine_if_person_is_current(doc) - if 'contact_details' in doc: - for contact_detail in doc['contact_details']: - if contact_detail['type'] == 'email': - contact, created = Contact.objects.get_or_create(popit_identifier=contact_detail['id'], - contact_type=contact_type, - writeitinstance=writeitinstance, - person=obj) - contact.value = contact_detail['value'] - contact.enabled = enable_contacts - contact.save() - created_emails.append(contact.value) - if 'email' in doc and doc['email'] is not None and doc['email'] not in created_emails: - contact, created = Contact.objects.get_or_create( - contact_type=contact_type, - writeitinstance=writeitinstance, - person=obj) - contact.value = doc['email'] - contact.enabled = enable_contacts - contact.save() - - -class PopitApiInstance(ApiInstance): - class Meta: - proxy = True - - def fetch_all_from_api(self, writeitinstance): - """ - Update all the local data from the API. This method actually delegates - to the other models. - """ - models = [PopitPerson] - for model in models: - model.fetch_all_from_api(instance=self, writeitinstance=writeitinstance) diff --git a/nuntium/tasks.py b/nuntium/tasks.py index 2f42a653..32bfeaae 100644 --- a/nuntium/tasks.py +++ b/nuntium/tasks.py @@ -1,7 +1,6 @@ from celery import task from .management.commands.send_mails import send_mails from instance.models import WriteitInstancePopitInstanceRecord -from nuntium.popit_api_instance import PopitApiInstance import logging diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index de6d8667..f53ed661 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -9,7 +9,6 @@ from django.conf import settings import re from django.utils.encoding import force_text -from nuntium.popit_api_instance import PopitApiInstance class InstanceResourceTestCase(ResourceTestCase): diff --git a/nuntium/tests/popit_api_instance_tests.py b/nuntium/tests/popit_api_instance_tests.py index 72418094..0d09a355 100644 --- a/nuntium/tests/popit_api_instance_tests.py +++ b/nuntium/tests/popit_api_instance_tests.py @@ -7,7 +7,6 @@ from contactos.models import Contact, ContactType from django.conf import settings from django.contrib.auth.models import User -from nuntium.popit_api_instance import is_current_membership class EmailCreationWhenPullingFromPopit(TestCase): diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 0cbce10a..53913f30 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -5,7 +5,6 @@ from django.utils.unittest import skip from django.contrib.auth.models import User from django.conf import settings -from nuntium.popit_api_instance import PopitApiInstance from datetime import timedelta from django.utils import timezone from mock import patch, call diff --git a/nuntium/tests/tasks_test.py b/nuntium/tests/tasks_test.py index 7cc9776e..b4eec7c5 100644 --- a/nuntium/tests/tasks_test.py +++ b/nuntium/tests/tasks_test.py @@ -5,7 +5,6 @@ from ..tasks import send_mails_task from mock import patch from popolo.models import Person -from nuntium.popit_api_instance import PopitApiInstance from django.contrib.auth.models import User from django.conf import settings from nuntium.tasks import pull_from_popit, update_all_popits diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index fd8a90b4..d874f126 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -11,7 +11,6 @@ from django.utils.translation import ugettext as _ from django.conf import settings from mock import patch -from nuntium.popit_api_instance import PopitApiInstance from requests.exceptions import ConnectionError from contactos.models import Contact, ContactType diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 4a2871a2..9d3383ba 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -12,7 +12,6 @@ from django.utils.translation import ugettext as _ from nuntium.user_section.forms import RelatePopitInstanceWithWriteItInstance from nuntium.management.commands.back_fill_writeit_popit_records import WPBackfillRecords -from nuntium.popit_api_instance import PopitApiInstance class RecreateWriteitInstancePopitInstanceRecord(UserSectionTestCase): diff --git a/nuntium/user_section/tests/user_section_views_tests.py b/nuntium/user_section/tests/user_section_views_tests.py index 48fae046..4cfa3e7e 100644 --- a/nuntium/user_section/tests/user_section_views_tests.py +++ b/nuntium/user_section/tests/user_section_views_tests.py @@ -13,7 +13,6 @@ NewAnswerNotificationTemplateForm, ConfirmationTemplateForm from django.test.utils import override_settings from urlparse import urlparse -from nuntium.popit_api_instance import PopitApiInstance import json from nuntium.user_section.views import WriteItInstanceCreateView diff --git a/nuntium/user_section/views.py b/nuntium/user_section/views.py index 12b3148d..b07ca4c2 100644 --- a/nuntium/user_section/views.py +++ b/nuntium/user_section/views.py @@ -29,7 +29,6 @@ from django.contrib import messages as view_messages from django.utils.translation import ugettext as _ import json -from nuntium.popit_api_instance import PopitApiInstance from nuntium.tasks import pull_from_popit from nuntium.user_section.forms import WriteItPopitUpdateForm from django.contrib.sites.models import Site From 6055d685febc92036aa82e4a45295b8089fcbc1e Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 11:26:46 +0100 Subject: [PATCH 21/51] Switch code from using ApiInstance to PopoloSource --- .../back_fill_writeit_popit_records.py | 8 +- nuntium/tests/instance_config_tests.py | 7 +- nuntium/tests/messages_test.py | 13 ++-- nuntium/tests/outbound_message_test.py | 3 +- nuntium/tests/popit_writeit_relation_tests.py | 75 ++++++++++--------- nuntium/tests/tasks_test.py | 19 ++--- nuntium/tests/writeitinstances_test.py | 25 ++++--- nuntium/user_section/forms.py | 2 +- .../tests/popit_instance_update_tests.py | 14 ++-- .../tests/user_section_views_tests.py | 5 +- nuntium/user_section/views.py | 1 + 11 files changed, 92 insertions(+), 80 deletions(-) diff --git a/nuntium/management/commands/back_fill_writeit_popit_records.py b/nuntium/management/commands/back_fill_writeit_popit_records.py index 62bfe230..0ba8ebed 100644 --- a/nuntium/management/commands/back_fill_writeit_popit_records.py +++ b/nuntium/management/commands/back_fill_writeit_popit_records.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand -from popit.models import ApiInstance from instance.models import WriteitInstancePopitInstanceRecord +from popolo_sources.models import PopoloSource from django.contrib.auth.models import User import logging @@ -11,11 +11,11 @@ class WPBackfillRecords(object): @classmethod def back_fill_popit_records(cls, writeitinstance): persons_in_instance = writeitinstance.persons.all() - api_instances = ApiInstance.objects.filter(person__in=persons_in_instance).distinct() - for a in api_instances: + popolo_sources = PopoloSource.objects.filter(persons__in=persons_in_instance).distinct() + for popolo_source in popolo_sources: record, created = WriteitInstancePopitInstanceRecord.objects.get_or_create( writeitinstance=writeitinstance, - popitapiinstance=a) + popolo_source=popolo_source) logger.info( u"Creating -> {record}\n".format(record=record.__unicode__())) diff --git a/nuntium/tests/instance_config_tests.py b/nuntium/tests/instance_config_tests.py index 4f936bbd..2d8555df 100644 --- a/nuntium/tests/instance_config_tests.py +++ b/nuntium/tests/instance_config_tests.py @@ -1,6 +1,7 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from instance.models import InstanceMembership, WriteItInstance, WriteItInstanceConfig +from instance.models import InstanceMembership, PopoloPerson, WriteItInstance, WriteItInstanceConfig +from popolo_sources.models import PopoloSource from nuntium.models import Message from popit.models import ApiInstance from popolo.models import Person @@ -13,8 +14,8 @@ class WriteItInstanceConfigTestCase(TestCase): def setUp(self): super(WriteItInstanceConfigTestCase, self).setUp() - self.api_instance1 = ApiInstance.objects.get(id=1) - self.api_instance2 = ApiInstance.objects.get(id=2) + self.popolo_source1 = PopoloSource.objects.get(id=1) + self.popolo_source2 = PopoloSource.objects.get(id=2) self.person1 = Person.objects.get(id=1) self.owner = User.objects.get(id=1) diff --git a/nuntium/tests/messages_test.py b/nuntium/tests/messages_test.py index e396fd91..8be8e695 100644 --- a/nuntium/tests/messages_test.py +++ b/nuntium/tests/messages_test.py @@ -5,7 +5,8 @@ from django.db import IntegrityError from django.utils.translation import ugettext as _ from contactos.models import Contact -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance +from popolo_sources.models import PopoloSource from ..models import Message, OutboundMessage, NoContactOM from popolo.models import Person from subdomains.utils import reverse @@ -468,7 +469,7 @@ class MysqlTesting(UsingDbMixin, OriginalTestCase): def setUp(self): super(MysqlTesting, self).setUp() user = User.objects.create_user(username='admin', password='a') - popit_instance = ApiInstance.objects.create( + popolo_source = PopoloSource.objects.create( url='http://popit.ciudadanointeligente.org', ) @@ -477,7 +478,8 @@ def setUp(self): slug='instance-1', owner=user, ) - self.person1 = Person.objects.create(name='Pedro', api_instance=popit_instance) + self.person1 = Person.objects.create(name='Pedro') + popolo_source.persons.add(self.person1) # This test was a bug against mysql def test_a_message_with_a_changed_slug(self): @@ -532,7 +534,7 @@ class PostgresTesting(UsingDbMixin, OriginalTestCase): def setUp(self): super(PostgresTesting, self).setUp() user = User.objects.create_user(username='admin', password='a') - popit_instance = ApiInstance.objects.create( + popolo_source = PopoloSource.objects.create( url='http://popit.ciudadanointeligente.org', ) @@ -540,7 +542,8 @@ def setUp(self): name='instance 1', slug='instance-1', owner=user) - self.person1 = Person.objects.create(name='Pedro', api_instance=popit_instance) + self.person1 = Person.objects.create(name='Pedro') + popolo_source.persons.add(self.person1) # This test was a bug against mysql def test_a_message_with_a_changed_slug(self): diff --git a/nuntium/tests/outbound_message_test.py b/nuntium/tests/outbound_message_test.py index 41325a35..9e3e8268 100644 --- a/nuntium/tests/outbound_message_test.py +++ b/nuntium/tests/outbound_message_test.py @@ -4,6 +4,7 @@ from django.utils.translation import ugettext as _ from contactos.models import Contact, ContactType from instance.models import WriteItInstance +from popolo_sources.models import PopoloSource from ..models import Message, OutboundMessage, \ MessageRecord, OutboundMessagePluginRecord, \ OutboundMessageIdentifier, Answer, \ @@ -21,7 +22,7 @@ class OutboundMessageTestCase(TestCase): def setUp(self): super(OutboundMessageTestCase, self).setUp() - self.api_instance1 = ApiInstance.objects.get(id=1) + self.popolo_source = PopoloSource.objects.get(id=1) self.contact1 = Contact.objects.get(id=1) self.message = Message.objects.get(id=1) diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 53913f30..40e66181 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -2,6 +2,7 @@ from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import ( InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) +from popolo_sources.models import PopoloSource from django.utils.unittest import skip from django.contrib.auth.models import User from django.conf import settings @@ -27,19 +28,19 @@ class PopitWriteitRelationRecord(TestCase): ''' def setUp(self): self.writeitinstance = WriteItInstance.objects.first() - self.api_instance = ApiInstance.objects.first() + self.popolo_source = PopoloSource.objects.first() self.owner = User.objects.first() def test_instanciate(self): '''Instanciate a WriteitInstancePopitInstanceRelation''' record = WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=self.writeitinstance, - popitapiinstance=self.api_instance, + popolo_source=self.popolo_source, ) self.assertTrue(record) self.assertEquals(record.writeitinstance, self.writeitinstance) - self.assertEquals(record.popitapiinstance, self.api_instance) + self.assertEquals(record.popolo_source, self.popolo_source) self.assertTrue(record.updated) self.assertTrue(record.created) self.assertEquals(record.status, 'nothing') @@ -50,7 +51,7 @@ def test_unicode(self): '''A WriteitInstancePopitInstanceRelation has a __unicode__ method''' record = WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=self.writeitinstance, - popitapiinstance=self.api_instance, + popolo_source=self.popolo_source, ) expected_unicode = "The people from http://popit.org/api/v1 were loaded into instance 1" self.assertEquals(record.__unicode__(), expected_unicode) @@ -58,12 +59,12 @@ def test_unicode(self): @popit_load_data() def test_it_does_not_try_to_replicate_the_memberships(self): '''This is related to issue #429''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # Doing it twice so I can replicate the bug - writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) - writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) + writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() @@ -74,10 +75,10 @@ def test_it_does_not_try_to_replicate_the_memberships(self): @popit_load_data() def test_clean_memberships(self): '''As part of bug #429 there can be several Membership between one writeitinstance and a person''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # there should be an amount of memberships - writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() previous_memberships = list(InstanceMembership.objects.filter(writeitinstance=writeitinstance)) @@ -87,7 +88,7 @@ def test_clean_memberships(self): InstanceMembership.objects.create(writeitinstance=writeitinstance, person=person) try: with popit_load_data(): - writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) except InstanceMembership.MultipleObjectsReturned, e: self.fail("There are more than one InstanceMembership " + e) @@ -110,11 +111,11 @@ def test_it_is_created_automatically_when_fetching_a_popit_instance(self): writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) - popit_instance = ApiInstance.objects.get(url=settings.TEST_POPIT_API_URL) + popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=writeitinstance, - popitapiinstance=popit_instance, + popolo_source=popolo_source, ) self.assertTrue(record) @@ -132,9 +133,9 @@ def test_what_if_the_url_doesnt_exist(self): non_existing_url = "http://nonexisting.url" writeitinstance.load_persons_from_a_popit_api(non_existing_url) - popit_instance_count = ApiInstance.objects.filter(url=non_existing_url).count() + popolo_source_count = PopoloSource.objects.filter(url=non_existing_url).count() - self.assertFalse(popit_instance_count) + self.assertFalse(popolo_source_count) @popit_load_data() def test_it_should_be_able_to_update_twice(self): @@ -148,13 +149,13 @@ def test_it_should_be_able_to_update_twice(self): writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) - popit_instance = ApiInstance.objects.get(url=settings.TEST_POPIT_API_URL) + popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=writeitinstance, - popitapiinstance=popit_instance, + popolo_source=popolo_source, ) self.assertNotEqual(record.created, record.updated) @@ -170,10 +171,10 @@ def test_it_should_update_the_date_every_time_it_is_updated(self): ) writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) - popit_instance = ApiInstance.objects.get(url=settings.TEST_POPIT_API_URL) + popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=writeitinstance, - popitapiinstance=popit_instance, + popolo_source=popolo_source, ) created_and_updated = timezone.now() - timedelta(days=2) @@ -190,7 +191,7 @@ def test_set_status(self): '''Setting the record status''' record = WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=self.writeitinstance, - popitapiinstance=self.api_instance, + popolo_source=self.popolo_source, ) record.set_status('error', 'Error 404') @@ -201,7 +202,7 @@ def test_set_status(self): @popit_load_data() def test_set_status_in_called_success(self): '''In progress and success status called''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) with patch.object(WriteitInstancePopitInstanceRecord, 'set_status', return_value=None) as set_status: @@ -230,14 +231,14 @@ def setUp(self): self.owner = self.writeitinstance.owner self.owner.set_password('feroz') self.owner.save() - self.popit_api_instance = PopitApiInstance.objects.first() + self.popolo_source = PopoloSource.objects.first() # Empty the popit_api_instance - self.popit_api_instance.person_set.all().delete() - self.popit_api_instance.url = settings.TEST_POPIT_API_URL - self.popit_api_instance.save() + self.popolo_source.persons.all().delete() + self.popolo_source.url = settings.TEST_POPIT_API_URL + self.popolo_source.save() self.popit_writeit_record = WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=self.writeitinstance, - popitapiinstance=self.popit_api_instance + popolo_source=self.popolo_source ) self.request_factory = RequestFactory() @@ -256,45 +257,45 @@ def setUp(self): def test_post_to_the_url_for_manual_resync(self): '''Resyncing can be done by posting to a url''' # This is just a symbolism but it is to show how this popit api is empty - self.assertFalse(self.popit_api_instance.person_set.all()) + self.assertFalse(self.popolo_source.persons.all()) url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': self.popit_api_instance.pk}) + 'popit_api_pk': self.popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = self.owner - response = ReSyncFromPopit.as_view()(request, popit_api_pk=self.popit_api_instance.pk) + response = ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) self.assertEquals(response.status_code, 200) # It should have been updated - self.assertTrue(self.popit_api_instance.person_set.all()) + self.assertTrue(self.popolo_source.persons.all()) @popit_load_data() def test_doesnt_add_another_relation_w_p(self): ''' - If a user posts to the server using another popit_api that has not previously been related + If a user posts to the server using another popolo source that has not previously been related it does not add another relation ''' - another_popit_api_instance = PopitApiInstance.objects.last() - self.assertNotIn(another_popit_api_instance, + another_popolo_source = PopoloSource.objects.last() + self.assertNotIn(another_popolo_source, self.writeitinstance.writeitinstancepopitinstancerecord_set.all()) url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': another_popit_api_instance.pk}) + 'popit_api_pk': another_popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = self.owner with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=another_popit_api_instance.pk) + ReSyncFromPopit.as_view()(request, popit_api_pk=another_popolo_source.pk) def test_post_has_to_be_the_owner_of_the_instance(self): '''Only the owner of an instance can resync''' url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': self.popit_api_instance.pk}) + 'popit_api_pk': self.popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = AnonymousUser() with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=self.popit_api_instance.pk) + ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) other_user = User.objects.create_user(username="other_user", password="s3cr3t0") @@ -302,7 +303,7 @@ def test_post_has_to_be_the_owner_of_the_instance(self): request.subdomain = self.writeitinstance.slug request.user = other_user with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=self.popit_api_instance.pk) + ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) from nuntium.user_section.views import WriteItPopitUpdateView diff --git a/nuntium/tests/tasks_test.py b/nuntium/tests/tasks_test.py index b4eec7c5..4fbd0f62 100644 --- a/nuntium/tests/tasks_test.py +++ b/nuntium/tests/tasks_test.py @@ -1,6 +1,7 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import WriteItInstance, WriteitInstancePopitInstanceRecord +from popolo_sources.models import PopoloSource from nuntium.models import OutboundMessage from ..tasks import send_mails_task from mock import patch @@ -71,7 +72,7 @@ def test_handling_unexpected_exceptions_in_send(self): class PullFromPopitTask(TestCase): def setUp(self): super(PullFromPopitTask, self).setUp() - self.api_instance1 = PopitApiInstance.objects.create(url=settings.TEST_POPIT_API_URL) + self.popolo_source = PopoloSource.objects.create(url=settings.TEST_POPIT_API_URL) self.person1 = Person.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -87,9 +88,9 @@ def test_do_the_pulling(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=writeitinstance, - popitapiinstance=self.api_instance1 + popolo_source=self.popolo_source ) - pull_from_popit.delay(writeitinstance, self.api_instance1) # Returns result + pull_from_popit.delay(writeitinstance, self.popolo_source) # Returns result self.assertTrue(writeitinstance.persons.all()) @@ -97,17 +98,17 @@ class PeriodicallyPullFromPopitClass(TestCase): def setUp(self): super(PeriodicallyPullFromPopitClass, self).setUp() self.owner = User.objects.get(id=1) - #this is the popit_api_instance created based on the previous load + #this is the popolo_source created based on the previous load self.writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - self.popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + self.popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=self.writeitinstance, - popitapiinstance=self.popit_api_instance + popolo_source=self.popolo_source ) #loading data from popit in a sync way with popit_load_data(): - self.writeitinstance._load_persons_from_a_popit_api(self.popit_api_instance) + self.writeitinstance._load_persons_from_a_popit_api(self.popolo_source) self.previously_created_persons = list(self.writeitinstance.persons.all()) @popit_load_data("other_persons") @@ -126,7 +127,7 @@ def test_it_does_not_autosync_if_disabled(self): '''If the instance has autosync disabled then it does not sync''' writeitinstance_popit_record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=self.writeitinstance, - popitapiinstance=self.popit_api_instance + popolo_source=self.popolo_source ) # Periodicity = 0 means that it is never going to send anything writeitinstance_popit_record.periodicity = 0 @@ -144,7 +145,7 @@ def test_autosyncs_receiving_a_parameter_with_the_periodicity(self): '''It can receive a parameter refering to the periodicity''' writeitinstance_popit_record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=self.writeitinstance, - popitapiinstance=self.popit_api_instance + popolo_source=self.popolo_source ) writeitinstance_popit_record.periodicity = '1D' # Daily writeitinstance_popit_record.save() diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index d874f126..e9bd2175 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -2,7 +2,8 @@ from urlparse import urlsplit, urlunsplit from global_test_case import GlobalTestCase as TestCase, popit_load_data from subdomains.utils import reverse -from instance.models import InstanceMembership, WriteItInstance +from instance.models import InstanceMembership, PopoloPerson, WriteItInstance +from popolo_sources.models import PopoloSource from nuntium.models import Message, Confirmation from popolo.models import Person from django.utils.unittest import skip @@ -18,8 +19,8 @@ class InstanceTestCase(TestCase): def setUp(self): super(InstanceTestCase, self).setUp() - self.api_instance1 = ApiInstance.objects.get(id=1) - self.api_instance2 = ApiInstance.objects.get(id=2) + self.popolo_source1 = PopoloSource.objects.get(id=1) + self.popolo_source2 = PopoloSource.objects.get(id=2) self.person1 = Person.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -128,7 +129,7 @@ def test_create_an_instance_and_load_persons_from_an_api(self): def test_it_uses_the_async_task_to_pull_people_from_popit(self): '''It uses the async task to pull people from popit''' writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) ''' I'm going to patch the method to know that it was run, I could do some other properties but I'm thinking that @@ -155,7 +156,7 @@ def test_it_has_a_pulling_from_popit_status(self): 'error': 0 }) - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) @popit_load_data() def test_can_create_messages(self): @@ -227,8 +228,8 @@ def test_writeitinstance_people_with_contacts(self): class WriteItInstanceLoadingPeopleFromAPopitApiTestCase(TestCase): def setUp(self): super(WriteItInstanceLoadingPeopleFromAPopitApiTestCase, self).setUp() - self.api_instance1 = ApiInstance.objects.get(id=1) - self.api_instance2 = ApiInstance.objects.get(id=2) + self.popolo_source1 = PopoloSource.objects.get(id=1) + self.popolo_source2 = PopoloSource.objects.get(id=2) self.person1 = Person.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -236,7 +237,7 @@ def setUp(self): @popit_load_data() def test_load_persons_from_a_popit_api(self): '''Loading persons from a popit api''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) @@ -251,7 +252,7 @@ def test_load_persons_from_a_popit_api(self): @popit_load_data() def test_it_returns_a_tuple(self): '''Returns a tuple''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=settings.TEST_POPIT_API_URL) + popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) result = writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) self.assertIsInstance(result, tuple) @@ -261,7 +262,7 @@ def test_it_returns_a_tuple(self): def test_it_returns_false_when_theres_a_problem(self): '''When there's a problem it returns false and the problem in the tuple''' non_existing_url = "http://nonexisting.url" - popit_api_instance = PopitApiInstance.objects.create(url=non_existing_url) + popit_api_instance = PopoloSource.objects.create(url=non_existing_url) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) result = writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) self.assertIsInstance(result, tuple) @@ -272,8 +273,8 @@ def test_it_returns_false_when_theres_a_problem(self): class InstanceDetailView(TestCase): def setUp(self): super(InstanceDetailView, self).setUp() - self.api_instance1 = ApiInstance.objects.get(id=1) - self.api_instance2 = ApiInstance.objects.get(id=2) + self.popolo_source1 = PopoloSource.objects.get(id=1) + self.popolo_source2 = PopoloSource.objects.get(id=2) self.person1 = Person.objects.get(id=1) self.writeitinstance1 = WriteItInstance.objects.get(id=1) self.url = self.writeitinstance1.get_absolute_url() diff --git a/nuntium/user_section/forms.py b/nuntium/user_section/forms.py index 11f915f8..845c480b 100644 --- a/nuntium/user_section/forms.py +++ b/nuntium/user_section/forms.py @@ -284,7 +284,7 @@ def relate(self): def clean(self, *args, **kwargs): cleaned_data = super(RelatePopitInstanceWithWriteItInstance, self).clean(*args, **kwargs) - if self.writeitinstance.writeitinstancepopitinstancerecord_set.filter(popitapiinstance__url=cleaned_data.get('popit_url')): + if self.writeitinstance.writeitinstancepopitinstancerecord_set.filter(popolo_source__url=cleaned_data.get('popit_url')): self.relate() raise ValidationError(_("You have already added this source. But we will fetch the data from it again now.")) return cleaned_data diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 9d3383ba..81091944 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -2,6 +2,7 @@ from subdomains.utils import reverse from instance.models import ( InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) +from popolo_sources.models import PopoloSource from django.contrib.auth.models import User from django.forms import Form, URLField from django.conf import settings @@ -28,10 +29,10 @@ def test_update_creates_records_given_an_instance(self): def test_creates_records_only_once(self): '''It creates the records only once''' w = WriteItInstance.objects.first() - a = ApiInstance.objects.first() + p = PopoloSource.objects.first() WriteitInstancePopitInstanceRecord.objects.create( writeitinstance=w, - popitapiinstance=a, + popolo_source=p, ) WPBackfillRecords.back_fill_popit_records(writeitinstance=w) records = WriteitInstancePopitInstanceRecord.objects.filter(writeitinstance=w) @@ -43,10 +44,11 @@ def test_update_creates_records_given_an_instance_2_persons(self): no matter if there are two persons related ''' w = WriteItInstance.objects.first() + popolo_source = w.persons.first().popolo_sources.first() another_person = Person.objects.create( - api_instance=w.persons.first().api_instance, - name="Another Person but with the same api Instance", + name="Another Person but with the same Popolo source", ) + another_person.popolo_sources.add(popolo_source) InstanceMembership.objects.create(writeitinstance=w, person=another_person) WPBackfillRecords.back_fill_popit_records(writeitinstance=w) records = WriteitInstancePopitInstanceRecord.objects.filter(writeitinstance=w) @@ -139,9 +141,9 @@ def test_it_parses_the_popit_api(self): def test_the_form_is_not_valid_if_there_is_another_popit(self): '''The form is not valid if there is already another popit api instance related''' - popit_api_instance = PopitApiInstance.objects.create(url=settings.TEST_POPIT_API_URL) + popolo_source = PopoloSource.objects.create(url=settings.TEST_POPIT_API_URL) WriteitInstancePopitInstanceRecord.objects.create(writeitinstance=self.writeitinstance, - popitapiinstance=popit_api_instance) + popolo_source=popolo_source) data = {"popit_url": settings.TEST_POPIT_API_URL} form = RelatePopitInstanceWithWriteItInstance(data=data, writeitinstance=self.writeitinstance) diff --git a/nuntium/user_section/tests/user_section_views_tests.py b/nuntium/user_section/tests/user_section_views_tests.py index 4cfa3e7e..fedf82cd 100644 --- a/nuntium/user_section/tests/user_section_views_tests.py +++ b/nuntium/user_section/tests/user_section_views_tests.py @@ -7,6 +7,7 @@ from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import WriteItInstance, WriteitInstancePopitInstanceRecord +from popolo_sources.models import PopoloSource from nuntium.user_section.views import WriteItInstanceUpdateView, WriteItInstanceApiDocsView from nuntium.user_section.forms import WriteItInstanceBasicForm, \ WriteItInstanceCreateForm, \ @@ -246,10 +247,10 @@ def setUp(self): super(WriteItInstancePullingDetailViewTestCase, self).setUp() self.writeitinstance = WriteItInstance.objects.get(id=1) self.owner = self.writeitinstance.owner - self.popit_api_instance = PopitApiInstance.objects.get(id=1) + self.popolo_source = PopoloSource.objects.get(id=1) self.record = WriteitInstancePopitInstanceRecord.objects.get_or_create( writeitinstance=self.writeitinstance, - popitapiinstance=self.popit_api_instance) + popolo_source=self.popolo_source) def test_theres_a_url_to_check_whats_happening(self): '''There is a url to let the user know what's happening with her/his instance''' diff --git a/nuntium/user_section/views.py b/nuntium/user_section/views.py index b07ca4c2..11d93e61 100644 --- a/nuntium/user_section/views.py +++ b/nuntium/user_section/views.py @@ -11,6 +11,7 @@ from mailit.forms import MailitTemplateForm from instance.models import WriteItInstance, WriteItInstanceConfig, WriteitInstancePopitInstanceRecord +from popolo_sources.models import PopoloSource from ..models import Message,\ NewAnswerNotificationTemplate, ConfirmationTemplate, \ Answer, Moderation, \ From d1a592cf36768de300c8491f6ab7bb513eadb9d0 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:55:07 +0100 Subject: [PATCH 22/51] Change the test PopIt API URL to point to the export.json endpoint --- writeit/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeit/settings.py b/writeit/settings.py index 4104b0e7..40c3a130 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -317,7 +317,7 @@ TEST_POPIT_API_PORT = '3000' TEST_POPIT_API_SUBDOMAIN = 'popit-django-test' -TEST_POPIT_API_URL = "http://%s.%s.xip.io:%s/api" % ( +TEST_POPIT_API_URL = "http://%s.%s.xip.io:%s/api/v0.1/export.json" % ( TEST_POPIT_API_SUBDOMAIN, TEST_POPIT_API_HOST_IP, TEST_POPIT_API_PORT, From 28f0be8cb3b89a0f2cc39678ba31e62b86b69251 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:03:46 +0100 Subject: [PATCH 23/51] Make WriteitInstancePopitInstanceRecord.__unicode__ use PopoloSource URL --- instance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instance/models.py b/instance/models.py index 1f4f5dcc..b3da198d 100644 --- a/instance/models.py +++ b/instance/models.py @@ -238,7 +238,7 @@ class WriteitInstancePopitInstanceRecord(models.Model): def __unicode__(self): return "The people from {url} were loaded into {instance}".format( - url=self.popitapiinstance.url, + url=self.popolo_source.url, instance=self.writeitinstance.__unicode__(), ) From a0c2707c9e056f0f392a526b80408d81800872e1 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 21 Sep 2016 11:12:36 +0100 Subject: [PATCH 24/51] Allow WriteItInstance.add_person to take the base Person model --- instance/models.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/instance/models.py b/instance/models.py index b3da198d..89d103ce 100644 --- a/instance/models.py +++ b/instance/models.py @@ -1,3 +1,4 @@ +from copy import copy import datetime from django.conf import settings @@ -27,6 +28,19 @@ class Meta: LinkToPopoloSource, related_query_name='people') + @classmethod + def create_from_base_instance(cls, person): + # Sometimes we have an instance of the base Person model, but + # want the proxy model instance instead for its helper methods + # or to add to a PopoloPerson relation. This seems like a + # pretty horrible way to do it, but I don't know a neater way + # to do this without something like + # PopoloPerson.objects.get(pk=person.id) which would incur a + # database query. http://stackoverflow.com/q/18473850/223092 + person = copy(person) + person.__class__ = cls + return person + @property def popolo_source(self): ct = ContentType.objects.get_for_model(Person) @@ -101,6 +115,10 @@ class WriteItInstance(models.Model): owner = models.ForeignKey(User, related_name="writeitinstances") def add_person(self, person): + # If we get given the base class rather than the proxy + # class, we have to convert it before assigning it. + if person._meta.model.__name__ == 'Person': + person = PopoloPerson.create_from_base_instance(person) InstanceMembership.objects.get_or_create( writeitinstance=self, person=person) From 7966ffa7ec3a6895287ae1f3cd07f6401bd43cc9 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:59:22 +0100 Subject: [PATCH 25/51] Update the code to sync from Popolo JSON, not a PopIt instance This also involves renaming the celery tasks and many methods that referred to PopIt explicitly so that they no longer do. --- instance/models.py | 178 +++++++++++++++--- nuntium/api.py | 2 +- nuntium/forms.py | 2 +- nuntium/tasks.py | 14 +- nuntium/tests/popit_api_instance_tests.py | 22 +-- nuntium/tests/popit_writeit_relation_tests.py | 24 +-- nuntium/tests/tasks_test.py | 16 +- .../tests/writeitinstance_newform_tests.py | 2 +- nuntium/tests/writeitinstances_test.py | 18 +- nuntium/user_section/forms.py | 2 +- nuntium/user_section/views.py | 8 +- writeit/settings.py | 6 +- 12 files changed, 210 insertions(+), 84 deletions(-) diff --git a/instance/models.py b/instance/models.py index 89d103ce..f6e4ce50 100644 --- a/instance/models.py +++ b/instance/models.py @@ -6,17 +6,24 @@ from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.models import ContentType from django.core import mail -from django.db import models +from django.db import models, transaction from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ from annoying.fields import AutoOneToOneField from autoslug import AutoSlugField -from popolo_sources.models import PopoloSource -from popolo.models import Person +from popolo.models import ( + Identifier as PopoloIdentifier, + Person, +) +from popolo_sources.models import LinkToPopoloSource, PopoloSource +from popolo_sources.importer import PopoloSourceImporter from requests.exceptions import ConnectionError from subdomains.utils import reverse +from contactos.models import Contact +from mailit import MailChannel + class PopoloPerson(Person): class Meta: @@ -102,6 +109,99 @@ def uri_for_api(self): msg = "Could find no global identifier for PopoloPerson with ID {0}" raise PopoloIdentifier.DoesNotExist(msg.format(self.pk)) +def today_in_date_range(start_date, end_date): + today = str(datetime.date.today()) + is_valid = ((not start_date) or str(start_date) <= today) and \ + ((not end_date) or str(end_date) >= today) + return is_valid + +def determine_if_person_is_current(person_object): + return any( + today_in_date_range(membership.start_date, membership.end_date) + for membership in person_object.memberships.all() + ) + +def create_contactos(person_object, writeitinstance): + """Update Contact objects from contactos from the django-popolo Person""" + + # This replicates the logic from nuntium/popit_api_instance.py to + # create these contacts, complete with its bugs, e.g. not removing + # old Contacts that are no longer current (FIXME) + contact_type = MailChannel().get_contact_type() + created_emails = set() + enable_contacts = determine_if_person_is_current(person_object) + for contact_detail in person_object.contact_details.all(): + if contact_detail.contact_type == 'email': + contact, created = Contact.objects.get_or_create( + contact_type=contact_type, + writeitinstance=writeitinstance, + person=person_object) + contact.value = contact_detail.value + contact.enabled = enable_contacts + contact.save() + created_emails.add(contact.value) + if person_object.email and person_object.email not in created_emails: + contact, created = Contact.objects.get_or_create( + contact_type=contact_type, + writeitinstance=writeitinstance, + person=person_object) + contact.value = person_object.email + contact.enabled = enable_contacts + contact.save() + + +class ExtraIdentifierCreator(object): + + def __init__(self, popolo_source): + self.popolo_source = popolo_source + + def notify(self, collection, django_object, created, popolo_data): + if collection == 'person': + uri = "{base_url}#{collection}-{popolo_id}".format( + base_url=self.popolo_source.url, + collection=collection, + popolo_id=popolo_data['id'], + ) + django_object.identifiers.create( + scheme='popolo_uri', + identifier=uri) + + def notify_deleted(self, collection, django_object): + pass + + +class PersonTracker(object): + + def __init__(self): + self.persons_present = set() + self.persons_deleted = set() + + def notify(self, collection, django_object, created, popolo_data): + if collection == 'person': + self.persons_present.add(django_object) + + def notify_deleted(self, collection, django_object): + if collection == 'person': + self.persons_deleted.add(django_object) + + +class InstanceMembershipUpdater(object): + + def __init__(self, writeitinstance): + self.writeitinstance = writeitinstance + + def notify(self, collection, django_object, created, popolo_data): + # Make sure the InstanceMembership exists exactly once: + if collection == 'person': + self.writeitinstance.add_person(django_object) + + def notify_deleted(self, collection, django_object): + # Make sure the InstanceMembership is removed: + if collection == 'person': + InstanceMembership.objects.filter( + writeitinstance=self.writeitinstance, + person_id=django_object.id) + class WriteItInstance(models.Model): """WriteItInstance: Entity that groups messages and people @@ -115,47 +215,71 @@ class WriteItInstance(models.Model): owner = models.ForeignKey(User, related_name="writeitinstances") def add_person(self, person): + """Ensure there's exactly one link between the instance and a person""" + # If we get given the base class rather than the proxy # class, we have to convert it before assigning it. if person._meta.model.__name__ == 'Person': person = PopoloPerson.create_from_base_instance(person) - InstanceMembership.objects.get_or_create( - writeitinstance=self, person=person) + with transaction.atomic(): + # There might be multiple InstanceMembership relationships + # between a particular person and instance, as a result of + # an earlier bug (#429), so when adding make sure there's + # only a single such InstanceMembership: + kwargs = {'writeitinstance': self, 'person': person} + existing = InstanceMembership.objects.filter(**kwargs) + existing_count = existing.count() + if existing_count == 0: + InstanceMembership.objects.create(**kwargs) + else: + to_keep = existing.first() + # Remove any extraneous additional InstanceMembership + # objects: + existing.exclude(pk=to_keep.id).delete() @property def persons_with_contacts(self): return self.persons.filter(contact__writeitinstance=self, contact__isnull=False).distinct() - def relate_with_persons_from_popit_api_instance(self, popit_api_instance): + def relate_with_persons_from_popolo_json(self, popolo_source): try: - popit_api_instance.fetch_all_from_api(writeitinstance=self) + importer = PopoloSourceImporter( + popolo_source, id_prefix='popolo:', truncate='yes') + importer.add_observer(ExtraIdentifierCreator(popolo_source)) + importer.add_observer(InstanceMembershipUpdater(self)) + person_tracker = PersonTracker() + importer.add_observer(person_tracker) + importer.update_from_source() + # Now update the contacts - note that we can't do this in + # an observer, because when you're notified of updates to + # a person their memberships won't be up to date yet - we + # need to wait until the end. + for person in person_tracker.persons_present: + create_contactos(person, self) + for person in person_tracker.persons_deleted: + # Make sure any Contact objects for people who have been + # deleted are disabled: + Contact.objects.filter( + writeitinstance=self.writeitinstance, + person=person).update(enabled=False) except ConnectionError, e: - self.do_something_with_a_vanished_popit_api_instance(popit_api_instance) + self.do_something_with_a_vanished_popit_api_instance(popolo_source) e.message = _('We could not connect with the URL') return (False, e) except Exception, e: - self.do_something_with_a_vanished_popit_api_instance(popit_api_instance) + self.do_something_with_a_vanished_popit_api_instance(popolo_source) return (False, e) - persons = Person.objects.filter(api_instance=popit_api_instance) - for person in persons: - # There could be several memberships created. - memberships = InstanceMembership.objects.filter(writeitinstance=self, person=person) - if memberships.count() == 0: - InstanceMembership.objects.create(writeitinstance=self, person=person) - if memberships.count() > 1: - membership = memberships[0] - memberships.exclude(id=membership.id).delete() return (True, None) - def do_something_with_a_vanished_popit_api_instance(self, popit_api_instance): + def do_something_with_a_vanished_popit_api_instance(self, popolo_source): pass - def _load_persons_from_a_popit_api(self, popit_api_instance): - success_relating_people, error = self.relate_with_persons_from_popit_api_instance(popit_api_instance) + def _load_persons_from_popolo_json(self, popolo_source): + success_relating_people, error = self.relate_with_persons_from_popolo_json(popolo_source) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=self, - popitapiinstance=popit_api_instance + popolo_source=popolo_source ) if success_relating_people: record.set_status('success') @@ -166,19 +290,19 @@ def _load_persons_from_a_popit_api(self, popit_api_instance): record.set_status('error', error.message) return (success_relating_people, error) - def load_persons_from_a_popit_api(self, popit_url): + def load_persons_from_popolo_json(self, popolo_json_url): '''This is an async wrapper for getting people from the api''' - popit_api_instance, created = PopitApiInstance.objects.get_or_create(url=popit_url) + popolo_source, created = PopoloSource.objects.get_or_create(url=popolo_json_url) record, created = WriteitInstancePopitInstanceRecord.objects.get_or_create( writeitinstance=self, - popitapiinstance=popit_api_instance + popolo_source=popolo_source ) if not created: record.updated = datetime.datetime.today() record.save() record.set_status('inprogress') - from nuntium.tasks import pull_from_popit - return pull_from_popit.delay(self, popit_api_instance) + from nuntium.tasks import pull_from_popolo_json + return pull_from_popolo_json.delay(self, popolo_source) def get_absolute_url(self): return reverse('instance_detail', subdomain=self.slug) diff --git a/nuntium/api.py b/nuntium/api.py index d3f84679..5a167eb5 100644 --- a/nuntium/api.py +++ b/nuntium/api.py @@ -104,7 +104,7 @@ def obj_create(self, bundle): bundle = super(WriteItInstanceResource, self).obj_create(bundle) instance = bundle.obj if "popit-api" in bundle.data and bundle.data["popit-api"]: - instance.load_persons_from_a_popit_api(bundle.data["popit-api"]) + instance.load_persons_from_popolo_json(bundle.data["popit-api"]) return bundle diff --git a/nuntium/forms.py b/nuntium/forms.py index 0017c2e9..be316192 100644 --- a/nuntium/forms.py +++ b/nuntium/forms.py @@ -174,7 +174,7 @@ class Meta: def relate_with_people(self): if self.cleaned_data['popit_url']: popit_url = self.cleaned_data['popit_url'] - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( popit_url ) diff --git a/nuntium/tasks.py b/nuntium/tasks.py index 32bfeaae..e05a2e25 100644 --- a/nuntium/tasks.py +++ b/nuntium/tasks.py @@ -1,6 +1,7 @@ from celery import task from .management.commands.send_mails import send_mails from instance.models import WriteitInstancePopitInstanceRecord +from popolo_sources.models import PopoloSource import logging @@ -13,16 +14,17 @@ def send_mails_task(): @task() -def pull_from_popit(writeitinstance, popit_api_instance): - result = writeitinstance._load_persons_from_a_popit_api(popit_api_instance) - logger.info(u'Resyncing ' + writeitinstance.__unicode__() + u' with ' + popit_api_instance.__unicode__()) +def pull_from_popolo_json(writeitinstance, popolo_source): + result = writeitinstance._load_persons_from_popolo_json(popolo_source) + logger.info(u'Resyncing {0} with {1}'.format( + writeitinstance, popolo_source)) return result @task() -def update_all_popits(periodicity='1W'): +def update_all_instances(periodicity='1W'): all_records = WriteitInstancePopitInstanceRecord.objects.filter(periodicity=periodicity) logger.info(u'Complete resync of all instances') for record in all_records: - popit_api_instance = PopitApiInstance.objects.get(id=record.popitapiinstance.id) - pull_from_popit(record.writeitinstance, popit_api_instance) + popolo_source = PopoloSource.objects.get(id=record.popolo_source.id) + pull_from_popolo_json(record.writeitinstance, popolo_source) diff --git a/nuntium/tests/popit_api_instance_tests.py b/nuntium/tests/popit_api_instance_tests.py index 0d09a355..0d60738d 100644 --- a/nuntium/tests/popit_api_instance_tests.py +++ b/nuntium/tests/popit_api_instance_tests.py @@ -18,7 +18,7 @@ def setUp(self): def test_it_pulls_and_creates_contacts(self): '''When pulling from popit it also creates emails''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) User.objects.create_user(username="perro", password="gato") @@ -35,10 +35,10 @@ def test_it_pulls_and_creates_contacts(self): @popit_load_data(fixture_name='persons_with_emails') def test_it_does_not_replicate_contacts(self): '''It does not replicate a contact several times''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) User.objects.create_user(username="perro", password="gato") @@ -51,13 +51,13 @@ def test_not_replicate_contact_even_if_value_changes(self): '''The value of an email has changed in popit but in writeit it should just update the value''' # Creating and loading the data with popit_load_data(fixture_name='persons_with_emails'): - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) # Updating the data and loading again with popit_load_data(fixture_name='persons_with_emails2'): - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) @@ -68,7 +68,7 @@ def test_not_replicate_contact_even_if_value_changes(self): @popit_load_data(fixture_name='other_people_with_popolo_emails') def test_get_emails_in_the_popolo_format(self): '''Get emails contact if it comes in the popolo format''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) fiera = self.instance.persons.filter(name="Fiera Feroz") @@ -83,10 +83,10 @@ def test_get_emails_in_the_popolo_format(self): @popit_load_data(fixture_name='other_people_with_popolo_emails') def test_get_twice_from_popit_does_not_repeat_the_email(self): '''Ít does not duplicate emails if they are comming in the field preferred email''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) fiera = self.instance.persons.filter(name="Fiera Feroz") @@ -96,7 +96,7 @@ def test_get_twice_from_popit_does_not_repeat_the_email(self): @popit_load_data(fixture_name='persons_with_null_values') def test_accept_null_values(self): '''It can process information that has null values''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) fiera = self.instance.persons.get(name="Fiera Feroz") @@ -106,7 +106,7 @@ def test_accept_null_values(self): def test_bug_506(self): '''If the same email is in preferred email and in the list of contact_details it creates a single one''' - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) @@ -128,7 +128,7 @@ def test_if_memberships_are_no_longer_active(self): mock_datetime.today.return_value = datetime(2015, 1, 1) mock_datetime.strptime = lambda *args, **kw: datetime.strptime(*args, **kw) - self.instance.load_persons_from_a_popit_api( + self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL ) # Benito was the boss between 1987-03-21 diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 40e66181..7d333a6a 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -63,8 +63,8 @@ def test_it_does_not_try_to_replicate_the_memberships(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # Doing it twice so I can replicate the bug - writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) - writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) + writeitinstance.relate_with_persons_from_popolo_json(popolo_source) + writeitinstance.relate_with_persons_from_popolo_json(popolo_source) amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() @@ -78,7 +78,7 @@ def test_clean_memberships(self): popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # there should be an amount of memberships - writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) + writeitinstance.relate_with_persons_from_popolo_json(popolo_source) amount_of_memberships = InstanceMembership.objects.filter(writeitinstance=writeitinstance).count() previous_memberships = list(InstanceMembership.objects.filter(writeitinstance=writeitinstance)) @@ -88,7 +88,7 @@ def test_clean_memberships(self): InstanceMembership.objects.create(writeitinstance=writeitinstance, person=person) try: with popit_load_data(): - writeitinstance.relate_with_persons_from_popit_api_instance(popolo_source) + writeitinstance.relate_with_persons_from_popolo_json(popolo_source) except InstanceMembership.MultipleObjectsReturned, e: self.fail("There are more than one InstanceMembership " + e) @@ -109,7 +109,7 @@ def test_it_is_created_automatically_when_fetching_a_popit_instance(self): owner=self.owner, ) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) @@ -132,7 +132,7 @@ def test_what_if_the_url_doesnt_exist(self): ) non_existing_url = "http://nonexisting.url" - writeitinstance.load_persons_from_a_popit_api(non_existing_url) + writeitinstance.load_persons_from_popolo_json(non_existing_url) popolo_source_count = PopoloSource.objects.filter(url=non_existing_url).count() self.assertFalse(popolo_source_count) @@ -147,11 +147,11 @@ def test_it_should_be_able_to_update_twice(self): owner=self.owner, ) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=writeitinstance, @@ -170,7 +170,7 @@ def test_it_should_update_the_date_every_time_it_is_updated(self): owner=self.owner, ) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) popolo_source = PopoloSource.objects.get(url=settings.TEST_POPIT_API_URL) record = WriteitInstancePopitInstanceRecord.objects.get( writeitinstance=writeitinstance, @@ -182,7 +182,7 @@ def test_it_should_update_the_date_every_time_it_is_updated(self): record.created = created_and_updated record.save() - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) record_again = WriteitInstancePopitInstanceRecord.objects.get(id=record.id) self.assertNotEqual(record_again.updated, created_and_updated) self.assertEquals(record_again.created, created_and_updated) @@ -206,7 +206,7 @@ def test_set_status_in_called_success(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) with patch.object(WriteitInstancePopitInstanceRecord, 'set_status', return_value=None) as set_status: - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) calls = [call('inprogress'), call('success')] @@ -217,7 +217,7 @@ def test_set_status_in_called_error(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) non_existing_url = "http://nonexisting.url" with patch.object(WriteitInstancePopitInstanceRecord, 'set_status', return_value=None) as set_status: - writeitinstance.load_persons_from_a_popit_api(non_existing_url) + writeitinstance.load_persons_from_popolo_json(non_existing_url) calls = [call('inprogress'), call('error', _('We could not connect with the URL'))] diff --git a/nuntium/tests/tasks_test.py b/nuntium/tests/tasks_test.py index 4fbd0f62..c63effbf 100644 --- a/nuntium/tests/tasks_test.py +++ b/nuntium/tests/tasks_test.py @@ -8,7 +8,7 @@ from popolo.models import Person from django.contrib.auth.models import User from django.conf import settings -from nuntium.tasks import pull_from_popit, update_all_popits +from nuntium.tasks import pull_from_popolo_json, update_all_instances class TasksTestCase(TestCase): @@ -79,7 +79,7 @@ def setUp(self): def test_the_pulling_task_name(self): '''The pulling from Popit Task has a name''' - self.assertEquals(pull_from_popit.name, 'nuntium.tasks.pull_from_popit') + self.assertEquals(pull_from_popolo_json.name, 'nuntium.tasks.pull_from_popolo_json') @popit_load_data() def test_do_the_pulling(self): @@ -90,7 +90,7 @@ def test_do_the_pulling(self): writeitinstance=writeitinstance, popolo_source=self.popolo_source ) - pull_from_popit.delay(writeitinstance, self.popolo_source) # Returns result + pull_from_popolo_json.delay(writeitinstance, self.popolo_source) # Returns result self.assertTrue(writeitinstance.persons.all()) @@ -108,7 +108,7 @@ def setUp(self): ) #loading data from popit in a sync way with popit_load_data(): - self.writeitinstance._load_persons_from_a_popit_api(self.popolo_source) + self.writeitinstance._load_persons_from_popolo_json(self.popolo_source) self.previously_created_persons = list(self.writeitinstance.persons.all()) @popit_load_data("other_persons") @@ -117,7 +117,7 @@ def test_update_existing(self): # This means that if I run the task then it should update the persons # I'm running the weekly job by default - update_all_popits.delay() + update_all_instances.delay() persons_after_updating = list(self.writeitinstance.persons.all()) self.assertNotEquals(self.previously_created_persons, persons_after_updating) @@ -134,7 +134,7 @@ def test_it_does_not_autosync_if_disabled(self): writeitinstance_popit_record.save() # The record has been set to autosync False - update_all_popits.delay() + update_all_instances.delay() # Loading new data persons_after_updating = list(self.writeitinstance.persons.all()) # It should not have updated our writeit instance @@ -152,13 +152,13 @@ def test_autosyncs_receiving_a_parameter_with_the_periodicity(self): # Now because it is receiving the default value = '1W' # it should not pull anyone - update_all_popits.delay() + update_all_instances.delay() persons_after_updating = list(self.writeitinstance.persons.all()) self.assertEquals(self.previously_created_persons, persons_after_updating) # But If I tell the runner that I'm running the daily # process then it should change it - update_all_popits.delay(periodicity="1D") + update_all_instances.delay(periodicity="1D") persons_after_updating = list(self.writeitinstance.persons.all()) self.assertNotEquals(self.previously_created_persons, persons_after_updating) diff --git a/nuntium/tests/writeitinstance_newform_tests.py b/nuntium/tests/writeitinstance_newform_tests.py index 77e2436c..4b660171 100644 --- a/nuntium/tests/writeitinstance_newform_tests.py +++ b/nuntium/tests/writeitinstance_newform_tests.py @@ -62,7 +62,7 @@ def test_it_has_all_the_fields(self): def test_it_uses_popit_main_url_as_well(self): '''It accepts main popit url as well''' - with patch('instance.models.WriteItInstance.load_persons_from_a_popit_api') as method_load: + with patch('instance.models.WriteItInstance.load_persons_from_popolo_json') as method_load: data = { 'owner': self.user.id, 'popit_url': 'https://kenyan-politicians.popit.mysociety.org/', diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index e9bd2175..f1768a8e 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -116,7 +116,7 @@ def test_membership(self): def test_create_an_instance_and_load_persons_from_an_api(self): writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) self.assertEquals(writeitinstance.persons.all().count(), 2) @@ -135,8 +135,8 @@ def test_it_uses_the_async_task_to_pull_people_from_popit(self): I'm going to patch the method to know that it was run, I could do some other properties but I'm thinking that this is the easyest to know if the method was used. ''' - with patch('nuntium.tasks.pull_from_popit.delay') as async_pulling_from_popit: - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + with patch('nuntium.tasks.pull_from_popolo_json.delay') as async_pulling_from_popit: + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) async_pulling_from_popit.assert_called_with(writeitinstance, popit_api_instance) @popit_load_data() @@ -147,7 +147,7 @@ def test_it_has_a_pulling_from_popit_status(self): 'inprogress': 0, 'success': 0, 'error': 0}) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) self.assertEquals(writeitinstance.pulling_from_popit_status, { 'nothing': 0, @@ -164,7 +164,7 @@ def test_can_create_messages(self): self.assertTrue(writeitinstance.config.allow_messages_using_form) self.assertFalse(writeitinstance.can_create_messages) - writeitinstance.load_persons_from_a_popit_api(settings.TEST_POPIT_API_URL) + writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) email_type = ContactType.objects.get(id=1) person = Person.objects.get(id=1) @@ -235,11 +235,11 @@ def setUp(self): self.owner = User.objects.get(id=1) @popit_load_data() - def test_load_persons_from_a_popit_api(self): + def test_load_persons_from_popolo_json(self): '''Loading persons from a popit api''' popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + writeitinstance.relate_with_persons_from_popolo_json(popit_api_instance) self.assertEquals(writeitinstance.persons.all().count(), 2) @@ -254,7 +254,7 @@ def test_it_returns_a_tuple(self): '''Returns a tuple''' popit_api_instance, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - result = writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + result = writeitinstance.relate_with_persons_from_popolo_json(popit_api_instance) self.assertIsInstance(result, tuple) self.assertTrue(result[0]) self.assertIsNone(result[1]) @@ -264,7 +264,7 @@ def test_it_returns_false_when_theres_a_problem(self): non_existing_url = "http://nonexisting.url" popit_api_instance = PopoloSource.objects.create(url=non_existing_url) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) - result = writeitinstance.relate_with_persons_from_popit_api_instance(popit_api_instance) + result = writeitinstance.relate_with_persons_from_popolo_json(popit_api_instance) self.assertIsInstance(result, tuple) self.assertFalse(result[0]) self.assertIsInstance(result[1], ConnectionError) diff --git a/nuntium/user_section/forms.py b/nuntium/user_section/forms.py index 845c480b..fc059e10 100644 --- a/nuntium/user_section/forms.py +++ b/nuntium/user_section/forms.py @@ -277,7 +277,7 @@ def __init__(self, *args, **kwargs): super(RelatePopitInstanceWithWriteItInstance, self).__init__(*args, **kwargs) def relate(self): - result = self.writeitinstance.load_persons_from_a_popit_api( + result = self.writeitinstance.load_persons_from_popolo_json( self.cleaned_data['popit_url'] ) return result diff --git a/nuntium/user_section/views.py b/nuntium/user_section/views.py index 11d93e61..67b8fa5b 100644 --- a/nuntium/user_section/views.py +++ b/nuntium/user_section/views.py @@ -30,7 +30,7 @@ from django.contrib import messages as view_messages from django.utils.translation import ugettext as _ import json -from nuntium.tasks import pull_from_popit +from nuntium.tasks import pull_from_popolo_json from nuntium.user_section.forms import WriteItPopitUpdateForm from django.contrib.sites.models import Site @@ -521,11 +521,11 @@ def post(self, request, *args, **kwargs): writeitinstance = get_object_or_404(WriteItInstance, slug=self.request.subdomain, owner=self.request.user) - popits_previously_related = PopitApiInstance.objects.filter( + popolo_sources_previously_related = PopoloSource.objects.filter( writeitinstancepopitinstancerecord__writeitinstance=writeitinstance) - popit_api_instance = get_object_or_404(popits_previously_related, pk=kwargs['popit_api_pk']) - pull_from_popit.delay(writeitinstance, popit_api_instance) + popolo_source = get_object_or_404(popolo_sources_previously_related, pk=kwargs['popit_api_pk']) + pull_from_popolo_json.delay(writeitinstance, popolo_source) return HttpResponse() diff --git a/writeit/settings.py b/writeit/settings.py index 40c3a130..7b70b83c 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -347,7 +347,7 @@ }, # Resyncs popit every week 'resync-popit-apis-every-week': { - 'task': 'nuntium.tasks.pull_from_popit', + 'task': 'nuntium.tasks.pull_from_popolo_json', 'kwargs': { 'periodicity': '1W' }, @@ -355,7 +355,7 @@ }, # Resyncs popit every day 'resync-popit-apis-every-day': { - 'task': 'nuntium.tasks.pull_from_popit', + 'task': 'nuntium.tasks.pull_from_popolo_json', 'kwargs': { 'periodicity': '1D' }, @@ -363,7 +363,7 @@ }, # Resyncs popit twice every day 'resync-popit-apis-twice-every-day': { - 'task': 'nuntium.tasks.pull_from_popit', + 'task': 'nuntium.tasks.pull_from_popolo_json', 'kwargs': { 'periodicity': '2D' }, From d634aa22cb87b61250ceae02d67a1bcaf699af68 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 10:06:39 +0100 Subject: [PATCH 26/51] Update tests for whether a membership is current --- nuntium/tests/popit_api_instance_tests.py | 40 +++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/nuntium/tests/popit_api_instance_tests.py b/nuntium/tests/popit_api_instance_tests.py index 0d60738d..9a205418 100644 --- a/nuntium/tests/popit_api_instance_tests.py +++ b/nuntium/tests/popit_api_instance_tests.py @@ -3,7 +3,7 @@ from mock import patch from global_test_case import GlobalTestCase as TestCase, popit_load_data -from instance.models import WriteItInstance +from instance.models import WriteItInstance, today_in_date_range from contactos.models import Contact, ContactType from django.conf import settings from django.contrib.auth.models import User @@ -124,9 +124,8 @@ def test_if_memberships_are_no_longer_active(self): contacts should be disabled. Related to #419. ''' - with patch('nuntium.popit_api_instance.datetime') as mock_datetime: - mock_datetime.today.return_value = datetime(2015, 1, 1) - mock_datetime.strptime = lambda *args, **kw: datetime.strptime(*args, **kw) + with patch('instance.models.datetime') as mock_datetime: + mock_datetime.date.today.return_value = datetime(2015, 1, 1) self.instance.load_persons_from_popolo_json( settings.TEST_POPIT_API_URL @@ -152,24 +151,23 @@ def test_if_memberships_are_no_longer_active(self): class IsActiveTestCase(TestCase): def test_validating_if_a_membership_is_active(self): - with patch('nuntium.popit_api_instance.datetime') as mock_datetime: - mock_datetime.today.return_value = datetime(2000, 1, 1) - mock_datetime.strptime = lambda *args, **kw: datetime.strptime(*args, **kw) - - far_past = "1900-01-01" - past = "1999-01-01" - future = "2020-01-01" - far_future = "2525-01-01" - - self.assertFalse(is_current_membership({'start_date': far_past, 'end_date': past})) - self.assertFalse(is_current_membership({'end_date': past})) - self.assertTrue(is_current_membership({'start_date': past})) - self.assertTrue(is_current_membership({'start_date': past, 'end_date': future})) - self.assertTrue(is_current_membership({'end_date': future})) - self.assertFalse(is_current_membership({'start_date': future, 'end_date': far_future})) + with patch('instance.models.datetime') as mock_datetime: + mock_datetime.date.today.return_value = datetime(2000, 1, 1) + + far_past = datetime(1900, 1, 1) + past = datetime(1999, 1, 1) + future = datetime(2020, 1, 1) + far_future = datetime(2525, 1, 1) + + self.assertFalse(today_in_date_range(far_past, past)) + self.assertFalse(today_in_date_range(None, past)) + self.assertTrue(today_in_date_range(past, None)) + self.assertTrue(today_in_date_range(past, future)) + self.assertTrue(today_in_date_range(None, future)) + self.assertFalse(today_in_date_range(future, far_future)) # Handles empty strings as dates - self.assertTrue(is_current_membership({'start_date': past, 'end_date': ""})) + self.assertTrue(today_in_date_range(past, None)) # If there's neither a start date or an end date, that's current. - self.assertTrue(is_current_membership({})) + self.assertTrue(today_in_date_range(None, None)) From 0adf5e3fe85aee1f9c5a2540aeef5b487cac7273 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 22 Sep 2016 14:56:35 +0100 Subject: [PATCH 27/51] Switch the EveryPolitician country selector to use Popolo JSON URLs This depends on this commit: https://github.com/everypolitician/everypolitician-writeinpublic/commit/ae62cadf4ff9ff4ea83c386ed634f9e1a672d9a1 ... which has been deployed now. --- nuntium/templates/nuntium/create_new_writeitinstance.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuntium/templates/nuntium/create_new_writeitinstance.html b/nuntium/templates/nuntium/create_new_writeitinstance.html index a23a2b0c..9d0afc94 100644 --- a/nuntium/templates/nuntium/create_new_writeitinstance.html +++ b/nuntium/templates/nuntium/create_new_writeitinstance.html @@ -25,7 +25,7 @@

{% trans "Create your new Site" %}

data-slug="{{ country.slug }}" data-email-count="{{ country.people_with_contact_details }}" data-person-count="{{ country.person_count }}" - value="{{ country.url }}" + value="{{ country.popolo_json_master_url }}" >{{ country.country }} - {{ country.legislature }} {% endfor %} From 49e6a7597630a44acec5e452d842e2e30136fc82 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 11:27:37 +0100 Subject: [PATCH 28/51] Update a test to use a django-popolo Identifier to get a person's ID --- nuntium/tests/message_creation_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nuntium/tests/message_creation_tests.py b/nuntium/tests/message_creation_tests.py index 616e30a8..ac7d5cdc 100644 --- a/nuntium/tests/message_creation_tests.py +++ b/nuntium/tests/message_creation_tests.py @@ -24,7 +24,9 @@ def test_go_straight_to_draft_given_person_id(self): subdomain=self.writeitinstance.slug, kwargs={'step': 'draft'}) - response = self.client.get(url, {'person_id': self.person1.popit_id}) + person_id = self.person1.identifiers.get(scheme='popolo:person').identifier + + response = self.client.get(url, {'person_id': person_id}) self.assertRedirects(response, url2) response = self.client.get(url2) self.assertEquals(response.context['message']['persons'][0].id, self.person1.id) From ec4e71d96ec632a0fd53715dcd5167de005bf46b Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 25 Aug 2016 13:31:42 +0100 Subject: [PATCH 29/51] Switch to using the PopoloPerson proxy model This include a migration that changes some foreign keys to use the PopoloPerson proxy model instead of the base Person model. --- nuntium/api.py | 13 ++++---- .../0007_switch_fks_to_popoloperson.py | 29 +++++++++++++++++ nuntium/models.py | 3 +- nuntium/tests/answer_attachments_tests.py | 4 +-- nuntium/tests/answers_test.py | 6 ++-- nuntium/tests/api/answer_resource_test.py | 5 ++- nuntium/tests/api/instance_resource_test.py | 15 +++++---- nuntium/tests/api/message_resource_test.py | 11 +++---- nuntium/tests/confirmation_template_test.py | 18 +++++------ nuntium/tests/instance_config_tests.py | 6 ++-- nuntium/tests/message_creation_tests.py | 9 +++--- nuntium/tests/message_form_tests.py | 31 +++++++++---------- .../tests/messages_per_person_view_test.py | 9 +++--- nuntium/tests/messages_test.py | 9 +++--- nuntium/tests/subscribers_test.py | 9 +++--- nuntium/tests/writeitinstances_test.py | 21 ++++++------- .../tests/popit_instance_update_tests.py | 6 ++-- .../tests/user_section_views_tests.py | 2 ++ 18 files changed, 111 insertions(+), 95 deletions(-) create mode 100644 nuntium/migrations/0007_switch_fks_to_popoloperson.py diff --git a/nuntium/api.py b/nuntium/api.py index 5a167eb5..9fb1b4ec 100644 --- a/nuntium/api.py +++ b/nuntium/api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from tastypie.resources import ModelResource, ALL_WITH_RELATIONS, Resource -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from .models import Message, Answer, \ OutboundMessageIdentifier, OutboundMessage, Confirmation from tastypie.authentication import ApiKeyAuthentication @@ -10,7 +10,6 @@ from tastypie import fields from tastypie.exceptions import ImmediateHttpResponse from tastypie import http -from popolo.models import Person from contactos.models import Contact from tastypie.paginator import Paginator from django.http import Http404, HttpResponseBadRequest @@ -34,7 +33,7 @@ def get_offset_from_page(self): class PersonResource(ModelResource): class Meta: - queryset = Person.objects.all() + queryset = PopoloPerson.objects.all() resource_name = 'person' authentication = ApiKeyAuthentication() @@ -185,7 +184,7 @@ class Meta: def build_filters(self, filters=None, ignore_bad_filters=False): result = super(MessageResource, self).build_filters(filters, ignore_bad_filters) - queryset = Person.objects.all() + queryset = PopoloPerson.objects.all() if 'writeitinstance__exact' in result: queryset = result['writeitinstance__exact'].persons.all() person = None @@ -193,12 +192,12 @@ def build_filters(self, filters=None, ignore_bad_filters=False): try: person = queryset.get(id=filters['person']) except: - raise Http404("Person does not exist") + raise Http404("PopoloPerson does not exist") if 'person__popit_id' in filters: try: person = queryset.get(popit_id=filters['person__popit_id']) except ObjectDoesNotExist: - raise Http404("Person does not exist") + raise Http404("PopoloPerson does not exist") if person: result['person'] = person return result @@ -224,7 +223,7 @@ def hydrate(self, bundle): else: for popit_url in bundle.data['persons']: try: - person = Person.objects.get(popit_url=popit_url) + person = PopoloPerson.objects.get(popit_url=popit_url) persons.append(person) except ObjectDoesNotExist: pass diff --git a/nuntium/migrations/0007_switch_fks_to_popoloperson.py b/nuntium/migrations/0007_switch_fks_to_popoloperson.py new file mode 100644 index 00000000..dc91fe7b --- /dev/null +++ b/nuntium/migrations/0007_switch_fks_to_popoloperson.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import datetime +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('nuntium', '0006_auto_remove_old_popit_proxy_models'), + ('instance', '0010_add_popoloperson_proxy_and_change_related'), + ] + + operations = [ + migrations.AlterField( + model_name='answer', + name='person', + field=models.ForeignKey(to='instance.PopoloPerson'), + preserve_default=True, + ), + migrations.AlterField( + model_name='nocontactom', + name='person', + field=models.ForeignKey(to='instance.PopoloPerson'), + preserve_default=True, + ), + ] diff --git a/nuntium/models.py b/nuntium/models.py index 4952a5b7..da63ead7 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -5,7 +5,6 @@ from django.db import models from django.utils.translation import override, ugettext_lazy as _ from django.core.exceptions import ValidationError, ObjectDoesNotExist -from popolo.models import Person as PopoloPerson from contactos.models import Contact from .plugins import OutputPlugin from django.contrib.contenttypes.models import ContentType @@ -29,7 +28,7 @@ import os import codecs -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from writeit_utils import escape_dictionary_values diff --git a/nuntium/tests/answer_attachments_tests.py b/nuntium/tests/answer_attachments_tests.py index 7cf1f42d..d388c6cd 100644 --- a/nuntium/tests/answer_attachments_tests.py +++ b/nuntium/tests/answer_attachments_tests.py @@ -1,6 +1,6 @@ from global_test_case import GlobalTestCase as TestCase +from instance.models import PopoloPerson from nuntium.models import Message, Answer, AnswerAttachment -from popolo.models import Person from django.core.files import File from subdomains.utils import reverse @@ -9,7 +9,7 @@ class AnswerAttachmentsTest(TestCase): def setUp(self): super(AnswerAttachmentsTest, self).setUp() self.message = Message.objects.all()[0] - self.person = Person.objects.all()[0] + self.person = PopoloPerson.objects.all()[0] self.answer = Answer.objects.create( message=self.message, person=self.person, diff --git a/nuntium/tests/answers_test.py b/nuntium/tests/answers_test.py index e0ad51eb..75a3e809 100644 --- a/nuntium/tests/answers_test.py +++ b/nuntium/tests/answers_test.py @@ -1,6 +1,6 @@ from global_test_case import GlobalTestCase as TestCase from ..models import Message, Answer -from popolo.models import Person +from instance.models import PopoloPerson from django.utils.translation import ugettext as _ @@ -8,9 +8,9 @@ class AnswerTestCase(TestCase): def setUp(self): super(AnswerTestCase, self).setUp() self.message = Message.objects.get(id=1) - self.person = Person.objects.get(id=1) + self.person = PopoloPerson.objects.get(id=1) #There is no membership for this guy to any writeitInstance - self.person_not_in_the_instance = Person.objects.get(id=2) + self.person_not_in_the_instance = PopoloPerson.objects.get(id=2) def test_create_an_answer(self): answer = Answer.objects.create(message=self.message, diff --git a/nuntium/tests/api/answer_resource_test.py b/nuntium/tests/api/answer_resource_test.py index b75a8218..6852689e 100644 --- a/nuntium/tests/api/answer_resource_test.py +++ b/nuntium/tests/api/answer_resource_test.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- from django.core.management import call_command -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popolo.models import Person from ...api import AnswerResource from django.http import HttpRequest from ...models import Answer, Message @@ -93,7 +92,7 @@ def test_get_the_person_that_answered(self): def test_answer_ordering(self): """The answers are displayed from new to old""" - person = Person.objects.get(id=1) + person = PopoloPerson.objects.get(id=1) message = Message.objects.get(id=1) Answer.objects.all().delete() diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index f53ed661..0bc19b03 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- from django.core.management import call_command -from instance.models import WriteItInstance +from instance.models import WriteItInstance, PopoloPerson from ...models import Message, Confirmation from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popolo.models import Person from global_test_case import popit_load_data from django.conf import settings import re @@ -59,7 +58,7 @@ def test_get_persons_of_an_instance(self): response = self.api_client.get(url, data=self.data) instance = self.deserialize(response) self.assertIn('persons', instance) - pedro = Person.objects.get(id=1) + pedro = PopoloPerson.objects.get(id=1) self.assertIn(pedro.popit_url, instance['persons']) def test_create_a_new_instance(self): @@ -126,8 +125,8 @@ def test_create_and_pull_people_from_a_popit_api(self): instance = WriteItInstance.objects.get(id=match_id.group('id')) self.assertEquals(instance.persons.count(), 2) #this should not break - raton = Person.objects.get(name=u'Ratón Inteligente') - fiera = Person.objects.get(name=u"Fiera Feroz") + raton = PopoloPerson.objects.get(name=u'Ratón Inteligente') + fiera = PopoloPerson.objects.get(name=u"Fiera Feroz") self.assertIn(raton, [r for r in instance.persons.all()]) self.assertIn(fiera, [r for r in instance.persons.all()]) @@ -143,7 +142,7 @@ def setUp(self): self.data = {'format': 'json', 'username': self.user.username, 'api_key': self.user.api_key.key} # creating messages - self.pedro = Person.objects.get(id=1) + self.pedro = PopoloPerson.objects.get(id=1) self.writeitinstance.add_person(self.pedro) # Setting that the contact is related to self.writeitinstance rather than to the user self.contact = self.pedro.contact_set.all()[0] @@ -164,7 +163,7 @@ def setUp(self): self.message1.recently_confirmated() # Confirmating - self.marcel = Person.objects.get(id=2) + self.marcel = PopoloPerson.objects.get(id=2) self.message2 = Message.objects.create( content='Content 1', author_name='Felipe', @@ -221,7 +220,7 @@ def test_filter_by_persons_popit_id(self): def test_create_a_message_with_a_person_that_is_in_2_api_instances(self): api = PopitApiInstance.objects.create(id=3, url="https://popit.org/api/v1") - p = Person.objects.create(api_instance=api, + p = PopoloPerson.objects.create(api_instance=api, name="Otro", popit_url="https://popit.org/api/v1/persons/1", popit_id="52bc7asdasd34567" diff --git a/nuntium/tests/api/message_resource_test.py b/nuntium/tests/api/message_resource_test.py index 01bf31c1..5e13f16d 100644 --- a/nuntium/tests/api/message_resource_test.py +++ b/nuntium/tests/api/message_resource_test.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- from django.core.management import call_command -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from ...models import Message, Confirmation from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User -from popolo.models import Person from django.utils.encoding import force_text @@ -37,7 +36,7 @@ def test_get_list_of_messages(self): def test_only_listing_my_messages(self): not_me = User.objects.create_user(username='not_me', password='not_my_password') writeitinstance = WriteItInstance.objects.create(name=u"a test", slug=u"a-test", owner=not_me) - person1 = Person.objects.get(id=1) + person1 = PopoloPerson.objects.get(id=1) writeitinstance.add_person(person1) i_should_not_see_this_message = Message.objects.create( content='Content 1 Public message', @@ -77,7 +76,7 @@ def test_list_of_messages_is_ordered(self): """ The list of messages shown in the API is ordered by created date""" # Preparing the test Message.objects.all().delete() - person1 = Person.objects.get(id=1) + person1 = PopoloPerson.objects.get(id=1) # cleaning up the database before message1 = Message.objects.create( content=u'Content 1', @@ -137,7 +136,7 @@ def test_the_message_has_the_people_it_was_sent_to(self): self.assertIn('popit_url', person) self.assertIn( - Person.objects.get(id=person['id']), + PopoloPerson.objects.get(id=person['id']), message.people.all(), ) self.assertEquals(len(message_from_the_api['people']), message.people.count()) @@ -173,7 +172,7 @@ def test_create_a_new_message(self): def test_create_a_new_message_in_not_my_instance(self): not_me = User.objects.create_user(username='not_me', password='not_my_password') writeitinstance = WriteItInstance.objects.create(name=u"a test", slug=u"a-test", owner=not_me) - person1 = Person.objects.get(id=1) + person1 = PopoloPerson.objects.get(id=1) writeitinstance.add_person(person1) message_data = { 'author_name': 'Felipipoo', diff --git a/nuntium/tests/confirmation_template_test.py b/nuntium/tests/confirmation_template_test.py index 75318fb7..92d2e6b7 100644 --- a/nuntium/tests/confirmation_template_test.py +++ b/nuntium/tests/confirmation_template_test.py @@ -1,6 +1,7 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance +from popolo_sources.models import PopoloSource from ..models import Confirmation, send_confirmation_email from ..models import Message, ConfirmationTemplate from django.core import mail @@ -11,7 +12,6 @@ from django.test.utils import override_settings from contactos.models import Contact, ContactType -from popolo.models import Person import codecs import os @@ -131,10 +131,9 @@ def test_rendering_templates(self): name="Test WriteIt Instance", owner=owner, ) - recipient = Person.objects.create( - name='Test Person', - api_instance_id=1, - ) + popolo_source = PopoloSource.objects.get(pk=1) + recipient = PopoloPerson.objects.create(name='Test Person') + popolo_source.persons.add(recipient) contact_type = ContactType.objects.create( name='Contact Type Name', label_name='Contact Type Label', @@ -248,10 +247,9 @@ def setUp(self): name="Test WriteIt Instance", owner=owner, ) - recipient = Person.objects.create( - name='Test Person', - api_instance_id=1, - ) + popolo_source = PopoloSource.objects.get(pk=1) + recipient = PopoloPerson.objects.create(name='Test Person') + popolo_source.persons.add(recipient) contact_type = ContactType.objects.create( name='Contact Type Name', label_name='Contact Type Label', diff --git a/nuntium/tests/instance_config_tests.py b/nuntium/tests/instance_config_tests.py index 2d8555df..bdcf9794 100644 --- a/nuntium/tests/instance_config_tests.py +++ b/nuntium/tests/instance_config_tests.py @@ -3,8 +3,6 @@ from instance.models import InstanceMembership, PopoloPerson, WriteItInstance, WriteItInstanceConfig from popolo_sources.models import PopoloSource from nuntium.models import Message -from popit.models import ApiInstance -from popolo.models import Person from django.contrib.auth.models import User from mailit import MailChannel from contactos.models import Contact @@ -16,7 +14,7 @@ def setUp(self): super(WriteItInstanceConfigTestCase, self).setUp() self.popolo_source1 = PopoloSource.objects.get(id=1) self.popolo_source2 = PopoloSource.objects.get(id=2) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -77,7 +75,7 @@ def test_get_custom_mail_connection(self): class TestingModeTestCase(TestCase): def setUp(self): super(TestingModeTestCase, self).setUp() - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) self.owner.email = "owner@ciudadanoi.org" diff --git a/nuntium/tests/message_creation_tests.py b/nuntium/tests/message_creation_tests.py index ac7d5cdc..da768401 100644 --- a/nuntium/tests/message_creation_tests.py +++ b/nuntium/tests/message_creation_tests.py @@ -1,9 +1,8 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from nuntium.models import Message from subdomains.utils import reverse -from popolo.models import Person from nuntium.forms import WhoForm, DraftForm @@ -11,8 +10,8 @@ class MessageCreationTestCase(TestCase): def setUp(self): super(MessageCreationTestCase, self).setUp() self.writeitinstance = WriteItInstance.objects.get(id=1) - self.person1 = Person.objects.get(id=1) - self.person2 = Person.objects.get(id=2) + self.person1 = PopoloPerson.objects.get(id=1) + self.person2 = PopoloPerson.objects.get(id=2) self.writeitinstance.add_person(self.person1) self.writeitinstance.add_person(self.person2) @@ -119,7 +118,7 @@ def test_go_through_the_whole_message_creation_process(self): def test_passing_non_valid_forms_across_instances(self): '''Described at #715''' first_instance = WriteItInstance.objects.first() - first_recipient = Person.objects.get(id=2) + first_recipient = PopoloPerson.objects.get(id=2) second_instance = WriteItInstance.objects.last() diff --git a/nuntium/tests/message_form_tests.py b/nuntium/tests/message_form_tests.py index f067ee8f..3ace28ac 100644 --- a/nuntium/tests/message_form_tests.py +++ b/nuntium/tests/message_form_tests.py @@ -1,8 +1,7 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase -from popolo.models import Person from contactos.models import Contact -from instance.models import InstanceMembership, WriteItInstance +from instance.models import InstanceMembership, PopoloPerson, WriteItInstance from ..models import Message, Confirmation, OutboundMessage from ..forms import MessageCreateForm, PersonMultipleChoiceField @@ -14,41 +13,41 @@ class PersonMultipleChoiceFieldTestCase(TestCase): def setUp(self): super(PersonMultipleChoiceFieldTestCase, self).setUp() - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) def test_get_widget(self): - field = PersonMultipleChoiceField(queryset=Person.objects.none()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.none()) widget = field.widget self.assertTrue(isinstance(widget, SelectMultiple)) def test_get_label_from_instance(self): - field = PersonMultipleChoiceField(queryset=Person.objects.all()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.all()) label = field.label_from_instance(self.person1) self.assertEquals(label, self.person1.name) def test_persons_without_a_contact(self): - felipe = Person.objects.get(name="Felipe") + felipe = PopoloPerson.objects.get(name="Felipe") felipe.contact_set.all().delete() # This guy does not have any contacts - field = PersonMultipleChoiceField(queryset=Person.objects.all()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.all()) rendered_field = field.widget.render(name='oli', value=None) self.assertIn(">Felipe *", rendered_field) def test_persons_with_bounced_contact(self): - felipe = Person.objects.get(name="Felipe") + felipe = PopoloPerson.objects.get(name="Felipe") for contact in felipe.contact_set.all(): contact.is_bounced = True contact.save() # This guy does not have any good contacts - field = PersonMultipleChoiceField(queryset=Person.objects.all()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.all()) rendered_field = field.widget.render(name='oli', value=None) self.assertIn(">Felipe *", rendered_field) def test_persons_with_bounced_contact_unicode(self): - felipe = Person.objects.get(name="Felipe") + felipe = PopoloPerson.objects.get(name="Felipe") felipe.name = u"Felipe Álvarez" felipe.save() @@ -57,12 +56,12 @@ def test_persons_with_bounced_contact_unicode(self): contact.save() # This guy does not have any good contacts - field = PersonMultipleChoiceField(queryset=Person.objects.all()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.all()) rendered_field = field.widget.render(name='oli', value=None) self.assertIn(u">Felipe Álvarez *", rendered_field) def test_previously_selected_persons(self): - field = PersonMultipleChoiceField(queryset=Person.objects.all()) + field = PersonMultipleChoiceField(queryset=PopoloPerson.objects.all()) rendered_field = field.widget.render(name='oli', value=[3]) self.assertIn('', rendered_field) @@ -71,7 +70,7 @@ class MessageFormTestCase(TestCase): def setUp(self): super(MessageFormTestCase, self).setUp() self.writeitinstance1 = WriteItInstance.objects.get(id=1) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.contact1 = Contact.objects.get(id=1) def test_form_fields(self): @@ -185,7 +184,7 @@ def test_maximum_recipients(self): '''When creating a message that includes more than maximum_recipients''' self.writeitinstance1.config.maximum_recipients = 1 self.writeitinstance1.config.save() - person2 = Person.objects.get(id=2) + person2 = PopoloPerson.objects.get(id=2) InstanceMembership.objects.create(writeitinstance=self.writeitinstance1, person=person2) data = { 'subject': u'Amor a la fiera', @@ -208,8 +207,8 @@ def setUp(self): self.writeitinstance1 = WriteItInstance.objects.get(id=1) self.writeitinstance1.config.rate_limiter = 1 self.writeitinstance1.config.save() - self.person1 = Person.objects.get(id=1) - self.person2 = Person.objects.get(id=2) + self.person1 = PopoloPerson.objects.get(id=1) + self.person2 = PopoloPerson.objects.get(id=2) self.message = Message.objects.get(id=1) Message.objects.create( content='Content 1', diff --git a/nuntium/tests/messages_per_person_view_test.py b/nuntium/tests/messages_per_person_view_test.py index bed35dbb..3c456a62 100644 --- a/nuntium/tests/messages_per_person_view_test.py +++ b/nuntium/tests/messages_per_person_view_test.py @@ -1,17 +1,16 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase from subdomains.utils import reverse -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from ..models import Message -from popolo.models import Person class MessagesPerPersonViewTestCase(TestCase): def setUp(self): super(MessagesPerPersonViewTestCase, self).setUp() self.writeitinstance = WriteItInstance.objects.get(id=1) - self.pedro = Person.objects.get(name="Pedro") - self.marcel = Person.objects.get(name="Marcel") + self.pedro = PopoloPerson.objects.get(name="Pedro") + self.marcel = PopoloPerson.objects.get(name="Marcel") def test_has_an_url(self): url = reverse( @@ -126,7 +125,7 @@ def test_person_non_existing_throws_500(self): self.assertEquals(response.status_code, 404) def test_the_person_does_exist_but_not_the_instance(self): - marcel = Person.objects.get(id=2) + marcel = PopoloPerson.objects.get(id=2) url = reverse( 'thread_to', subdomain='non-existing-slug', diff --git a/nuntium/tests/messages_test.py b/nuntium/tests/messages_test.py index 8be8e695..d535aa36 100644 --- a/nuntium/tests/messages_test.py +++ b/nuntium/tests/messages_test.py @@ -8,7 +8,6 @@ from instance.models import PopoloPerson, WriteItInstance from popolo_sources.models import PopoloSource from ..models import Message, OutboundMessage, NoContactOM -from popolo.models import Person from subdomains.utils import reverse from django.contrib.auth.models import User from django.template.defaultfilters import slugify @@ -20,8 +19,8 @@ class TestMessages(TestCase): def setUp(self): super(TestMessages, self).setUp() self.writeitinstance1 = WriteItInstance.objects.get(id=1) - self.person1 = Person.objects.get(id=1) - self.person2 = Person.objects.get(id=2) + self.person1 = PopoloPerson.objects.get(id=1) + self.person2 = PopoloPerson.objects.get(id=2) def test_create_message(self): message = Message.objects.create( @@ -478,7 +477,7 @@ def setUp(self): slug='instance-1', owner=user, ) - self.person1 = Person.objects.create(name='Pedro') + self.person1 = PopoloPerson.objects.create(name='Pedro') popolo_source.persons.add(self.person1) # This test was a bug against mysql @@ -542,7 +541,7 @@ def setUp(self): name='instance 1', slug='instance-1', owner=user) - self.person1 = Person.objects.create(name='Pedro') + self.person1 = PopoloPerson.objects.create(name='Pedro') popolo_source.persons.add(self.person1) # This test was a bug against mysql diff --git a/nuntium/tests/subscribers_test.py b/nuntium/tests/subscribers_test.py index 16dff878..57bdd17a 100644 --- a/nuntium/tests/subscribers_test.py +++ b/nuntium/tests/subscribers_test.py @@ -1,8 +1,7 @@ from global_test_case import GlobalTestCase as TestCase -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from ..models import Subscriber, Message, \ Confirmation, Answer, NewAnswerNotificationTemplate -from popolo.models import Person from django.contrib.auth.models import User from django.core import mail from django.conf import settings @@ -18,7 +17,7 @@ def setUp(self): super(SubscribersTestCase, self).setUp() self.moderation_not_needed_instance = WriteItInstance.objects.get(id=1) self.message = Message.objects.get(id=1) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) def test_create_a_new_subscriber(self): subscriber = Subscriber.objects.create(message=self.message, email='felipe@lab.ciudadanointeligente.org') @@ -68,7 +67,7 @@ def setUp(self): self.instance = WriteItInstance.objects.get(id=1) self.message = Message.objects.get(id=1) - self.pedro = Person.objects.get(id=1) + self.pedro = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) self.answer = Answer.objects.create( content="Ola ke ase? pedalea o ke ase?", @@ -126,7 +125,7 @@ def setUp(self): self.instance = WriteItInstance.objects.get(id=1) self.message = Message.objects.get(id=1) self.subscriber = Subscriber.objects.create(message=self.message, email=self.message.author_email) - self.pedro = Person.objects.get(id=1) + self.pedro = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) self.instance.new_answer_notification_template.subject_template = 'weeena pelao %(person)s %(message)s' self.instance.new_answer_notification_template.save() diff --git a/nuntium/tests/writeitinstances_test.py b/nuntium/tests/writeitinstances_test.py index f1768a8e..372ea5ca 100644 --- a/nuntium/tests/writeitinstances_test.py +++ b/nuntium/tests/writeitinstances_test.py @@ -5,7 +5,6 @@ from instance.models import InstanceMembership, PopoloPerson, WriteItInstance from popolo_sources.models import PopoloSource from nuntium.models import Message, Confirmation -from popolo.models import Person from django.utils.unittest import skip from django.contrib.auth.models import User from django.utils.translation import activate @@ -21,7 +20,7 @@ def setUp(self): super(InstanceTestCase, self).setUp() self.popolo_source1 = PopoloSource.objects.get(id=1) self.popolo_source2 = PopoloSource.objects.get(id=2) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -120,8 +119,8 @@ def test_create_an_instance_and_load_persons_from_an_api(self): self.assertEquals(writeitinstance.persons.all().count(), 2) - raton = Person.objects.get(name='Ratón Inteligente') - fiera = Person.objects.get(name="Fiera Feroz") + raton = PopoloPerson.objects.get(name='Ratón Inteligente') + fiera = PopoloPerson.objects.get(name="Fiera Feroz") self.assertIn(raton, [r for r in writeitinstance.persons.all()]) self.assertIn(fiera, [r for r in writeitinstance.persons.all()]) @@ -167,7 +166,7 @@ def test_can_create_messages(self): writeitinstance.load_persons_from_popolo_json(settings.TEST_POPIT_API_URL) email_type = ContactType.objects.get(id=1) - person = Person.objects.get(id=1) + person = PopoloPerson.objects.get(id=1) Contact.objects.create( person=person, contact_type=email_type, @@ -190,8 +189,8 @@ def setUp(self): name='instance 1', slug='instance-1', owner=self.owner) - self.person1 = Person.objects.get(id=1) - self.person2 = Person.objects.get(id=2) + self.person1 = PopoloPerson.objects.get(id=1) + self.person2 = PopoloPerson.objects.get(id=2) def test_instance_add_person(self): self.writeitinstance.add_person(self.person1) @@ -230,7 +229,7 @@ def setUp(self): super(WriteItInstanceLoadingPeopleFromAPopitApiTestCase, self).setUp() self.popolo_source1 = PopoloSource.objects.get(id=1) self.popolo_source2 = PopoloSource.objects.get(id=2) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.owner = User.objects.get(id=1) @@ -243,8 +242,8 @@ def test_load_persons_from_popolo_json(self): self.assertEquals(writeitinstance.persons.all().count(), 2) - raton = Person.objects.get(name='Ratón Inteligente') - fiera = Person.objects.get(name="Fiera Feroz") + raton = PopoloPerson.objects.get(name='Ratón Inteligente') + fiera = PopoloPerson.objects.get(name="Fiera Feroz") self.assertIn(raton, [r for r in writeitinstance.persons.all()]) self.assertIn(fiera, [r for r in writeitinstance.persons.all()]) @@ -275,7 +274,7 @@ def setUp(self): super(InstanceDetailView, self).setUp() self.popolo_source1 = PopoloSource.objects.get(id=1) self.popolo_source2 = PopoloSource.objects.get(id=2) - self.person1 = Person.objects.get(id=1) + self.person1 = PopoloPerson.objects.get(id=1) self.writeitinstance1 = WriteItInstance.objects.get(id=1) self.url = self.writeitinstance1.get_absolute_url() diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 81091944..9998bfaf 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -1,13 +1,13 @@ from global_test_case import popit_load_data from subdomains.utils import reverse from instance.models import ( - InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) + InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord, + PopoloPerson) from popolo_sources.models import PopoloSource from django.contrib.auth.models import User from django.forms import Form, URLField from django.conf import settings from django.core.management import call_command -from popolo.models import Person from django.utils.unittest import skip from user_section_views_tests import UserSectionTestCase from django.utils.translation import ugettext as _ @@ -45,7 +45,7 @@ def test_update_creates_records_given_an_instance_2_persons(self): ''' w = WriteItInstance.objects.first() popolo_source = w.persons.first().popolo_sources.first() - another_person = Person.objects.create( + another_person = PopoloPerson.objects.create( name="Another Person but with the same Popolo source", ) another_person.popolo_sources.add(popolo_source) diff --git a/nuntium/user_section/tests/user_section_views_tests.py b/nuntium/user_section/tests/user_section_views_tests.py index fedf82cd..fc2d4eb6 100644 --- a/nuntium/user_section/tests/user_section_views_tests.py +++ b/nuntium/user_section/tests/user_section_views_tests.py @@ -538,6 +538,8 @@ def test_create_an_instance_get_not_logged(self): response = c.get(url) self.assertRedirectToLogin(response) + # FIXME: this should mock + # http://everypolitician.github.io/everypolitician-writeinpublic/countries.json def test_get_url(self): '''Get the url for creating a new instance (it doesn't redrect anywhere)''' url = reverse('create_writeit_instance') From b979cb6e33f24857bed2fb8bc0e7cf716ee3af27 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:04:53 +0100 Subject: [PATCH 30/51] Re-record fixtures now that we're only using PopIt's export.json Rather than using the /persons collection endpoint of PopIt's API, the code now just uses export.json to extract Popolo JSON from PopIt. This commit has the results of re-recording these fixtures (using roughly the procedure here: https://gist.github.com/mhl/848f7980d256ca38ff603375be4f264e ). The exception was persons_with_null_values.yaml which seems to be missing the source JSON MongoDB fixture, so this was created by hand based on one of the others. --- .../tests/fixtures/vcr_cassettes/default.yaml | 112 +++--------------- .../other_people_with_popolo_emails.yaml | 60 +++------- .../fixtures/vcr_cassettes/other_persons.yaml | 60 +++------- ...th_preferred_email_and_contact_detail.yaml | 60 +++------- .../vcr_cassettes/persons_with_emails.yaml | 70 +++++++---- .../vcr_cassettes/persons_with_emails2.yaml | 36 +++--- .../persons_with_memberships.yaml | 60 +++------- .../persons_with_null_values.yaml | 66 ++--------- 8 files changed, 153 insertions(+), 371 deletions(-) diff --git a/nuntium/tests/fixtures/vcr_cassettes/default.yaml b/nuntium/tests/fixtures/vcr_cassettes/default.yaml index fe2611bb..da36cfec 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/default.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/default.yaml @@ -5,130 +5,54 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/about/" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { - "message": "Not Found", - "code": 404 + "message": "OK", + "code": 200 }, "headers": { - "date": [ - "Tue, 28 Apr 2015 19:40:19 GMT" - ], "content-length": [ - "56" - ], - "content-type": [ - "application/json; charset=utf-8" - ], - "connection": [ - "keep-alive" + "606" ], "x-powered-by": [ "Express" - ] - }, - "body": { - "string": "{\n \"errors\": [\n \"collection 'about' not found\"\n ]\n}" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" ], - "Accept-Encoding": [ - "gzip, deflate" + "connection": [ + "keep-alive" ], - "accept": [ - "application/json" + "etag": [ + "W/\"25e-1736e4de\"" ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { "date": [ - "Tue, 28 Apr 2015 19:40:19 GMT" + "Thu, 25 Aug 2016 13:19:29 GMT" ], - "content-length": [ - "622" + "access-control-allow-origin": [ + "*" ], "content-type": [ "application/json; charset=utf-8" - ], - "connection": [ - "keep-alive" - ], - "x-powered-by": [ - "Express" ] }, "body": { - "string": "{\n \"total\": 2,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/other_people_with_popolo_emails.yaml b/nuntium/tests/fixtures/vcr_cassettes/other_people_with_popolo_emails.yaml index e3b06fc2..cbb40eb2 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/other_people_with_popolo_emails.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/other_people_with_popolo_emails.yaml @@ -5,56 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -63,7 +28,7 @@ }, "headers": { "content-length": [ - "1057" + "1036" ], "x-powered-by": [ "Express" @@ -72,19 +37,22 @@ "keep-alive" ], "etag": [ - "\"574730498\"" + "W/\"mz6jW/xs2fju773ot3jpCw==\"" ], "date": [ - "Tue, 28 Apr 2015 20:37:45 GMT" + "Thu, 25 Aug 2016 13:25:02 GMT" + ], + "access-control-allow-origin": [ + "*" ], "content-type": [ "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 3,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanointeligente.org\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"email\": \"fiera-popolo@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"benito\",\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"email\": \"fiera-popolo@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/other_persons.yaml b/nuntium/tests/fixtures/vcr_cassettes/other_persons.yaml index 6d8d3fa1..feb7acf5 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/other_persons.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/other_persons.yaml @@ -5,56 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -63,7 +28,7 @@ }, "headers": { "content-length": [ - "1303" + "1282" ], "x-powered-by": [ "Express" @@ -72,19 +37,22 @@ "keep-alive" ], "etag": [ - "\"-1067133260\"" + "W/\"FPWHcogAxaI1qzqMIygmaQ==\"" ], "date": [ - "Tue, 28 Apr 2015 20:47:12 GMT" + "Thu, 25 Aug 2016 13:31:39 GMT" + ], + "access-control-allow-origin": [ + "*" ], "content-type": [ "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 3,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"benito\",\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/person_with_preferred_email_and_contact_detail.yaml b/nuntium/tests/fixtures/vcr_cassettes/person_with_preferred_email_and_contact_detail.yaml index 3bd0387d..b03bf059 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/person_with_preferred_email_and_contact_detail.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/person_with_preferred_email_and_contact_detail.yaml @@ -5,56 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -63,7 +28,7 @@ }, "headers": { "content-length": [ - "2897" + "1381" ], "x-powered-by": [ "Express" @@ -72,19 +37,22 @@ "keep-alive" ], "etag": [ - "\"799235944\"" + "W/\"HCQhDrQsqge62DOEHmKiqQ==\"" ], "date": [ - "Tue, 28 Apr 2015 19:56:12 GMT" + "Thu, 25 Aug 2016 13:28:32 GMT" + ], + "access-control-allow-origin": [ + "*" ], "content-type": [ "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 3,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanoi.org\",\n \"memberships\": [\n {\n \"id\": \"ratonNotBoss\",\n \"html_url\": \"https://ratonNotBoss\",\n \"url\": \"https://ratonNotBoss\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2045-03-20\",\n \"person_id\": \"raton-inteligente\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra muy buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [\n {\n \"id\": \"fieraBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"2011-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"links\": [],\n \"contact_details\": []\n },\n {\n \"id\": \"fieraNotBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"2005-03-21\",\n \"end_date\": \"2009-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"benito\",\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"memberships\": [\n {\n \"id\": \"benitoBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2007-03-20\",\n \"person_id\": \"benito\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [],\n \"posts\": [],\n \"memberships\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails.yaml b/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails.yaml index b7ee2862..f2d432d6 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails.yaml @@ -5,21 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -27,12 +27,30 @@ "code": 200 }, "headers": { + "content-length": [ + "806" + ], + "x-powered-by": [ + "Express" + ], + "connection": [ + "keep-alive" + ], + "etag": [ + "W/\"326-71692aee\"" + ], + "date": [ + "Thu, 25 Aug 2016 13:21:45 GMT" + ], + "access-control-allow-origin": [ + "*" + ], "content-type": [ "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } }, @@ -40,21 +58,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -62,26 +80,32 @@ "code": 200 }, "headers": { - "date": [ - "Tue, 28 Apr 2015 20:03:07 GMT" - ], "content-length": [ - "822" + "806" ], - "content-type": [ - "application/json; charset=utf-8" + "x-powered-by": [ + "Express" ], "connection": [ "keep-alive" ], - "x-powered-by": [ - "Express" + "etag": [ + "W/\"326-71692aee\"" + ], + "date": [ + "Thu, 25 Aug 2016 13:21:45 GMT" + ], + "access-control-allow-origin": [ + "*" + ], + "content-type": [ + "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 2,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails2.yaml b/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails2.yaml index 18531fb7..34309f4f 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails2.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/persons_with_emails2.yaml @@ -5,21 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -27,24 +27,30 @@ "code": 200 }, "headers": { - "date": [ - "Tue, 28 Apr 2015 19:56:16 GMT" - ], "content-length": [ - "818" + "802" ], - "content-type": [ - "application/json; charset=utf-8" + "x-powered-by": [ + "Express" ], "connection": [ "keep-alive" ], - "x-powered-by": [ - "Express" + "etag": [ + "W/\"322-8a4c1e31\"" + ], + "date": [ + "Thu, 25 Aug 2016 13:23:21 GMT" + ], + "access-control-allow-origin": [ + "*" + ], + "content-type": [ + "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 2,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fierita@votainteligente.cl\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fierita@votainteligente.cl\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } diff --git a/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml b/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml index 653191b2..3ab66064 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml @@ -5,56 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -63,7 +28,7 @@ }, "headers": { "content-length": [ - "2897" + "3153" ], "x-powered-by": [ "Express" @@ -72,19 +37,22 @@ "keep-alive" ], "etag": [ - "\"799235944\"" + "W/\"CZfM3c0Tcl07j8n43iW97Q==\"" ], "date": [ - "Tue, 28 Apr 2015 19:56:13 GMT" + "Thu, 25 Aug 2016 13:30:32 GMT" + ], + "access-control-allow-origin": [ + "*" ], "content-type": [ "application/json; charset=utf-8" ] }, "body": { - "string": "{\n \"total\": 3,\n \"page\": 1,\n \"per_page\": 50,\n \"has_more\": false,\n \"result\": [\n {\n \"id\": \"raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanoi.org\",\n \"memberships\": [\n {\n \"id\": \"ratonNotBoss\",\n \"html_url\": \"https://ratonNotBoss\",\n \"url\": \"https://ratonNotBoss\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2045-03-20\",\n \"person_id\": \"raton-inteligente\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra muy buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"memberships\": [\n {\n \"id\": \"fieraBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"2011-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"links\": [],\n \"contact_details\": []\n },\n {\n \"id\": \"fieraNotBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"2005-03-21\",\n \"end_date\": \"2009-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n },\n {\n \"id\": \"benito\",\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"memberships\": [\n {\n \"id\": \"benitoBoss\",\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2007-03-20\",\n \"person_id\": \"benito\",\n \"links\": [],\n \"contact_details\": []\n }\n ],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": []\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanoi.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra muy buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [\n {\n \"html_url\": \"https://\",\n \"url\": \"https://\",\n \"classification\": \"party\",\n \"id\": \"57bef28f7b68c342193dd2ea\",\n \"name\": \"Dogs fci\",\n \"images\": [],\n \"posts\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [\n {\n \"name\": \"Dogs\",\n \"id\": \"54e314e82ec9128d32084466\"\n }\n ]\n }\n ],\n \"memberships\": [\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"2011-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"2005-03-21\",\n \"end_date\": \"2009-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraNotBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2007-03-20\",\n \"person_id\": \"benito\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"benitoBoss\"\n },\n {\n \"html_url\": \"https://ratonNotBoss\",\n \"url\": \"https://ratonNotBoss\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2045-03-20\",\n \"person_id\": \"raton-inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"ratonNotBoss\"\n }\n ],\n \"posts\": []\n}" } } } ] -} +} \ No newline at end of file diff --git a/nuntium/tests/fixtures/vcr_cassettes/persons_with_null_values.yaml b/nuntium/tests/fixtures/vcr_cassettes/persons_with_null_values.yaml index aef92f44..ed03dc48 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/persons_with_null_values.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/persons_with_null_values.yaml @@ -5,56 +5,21 @@ "request": { "body": null, "headers": { - "content-type": [ - "application/json" - ], - "Accept-Encoding": [ - "gzip, deflate" - ], - "accept": [ - "application/json" - ], - "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" - ] - }, - "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api" - }, - "response": { - "status": { - "message": "OK", - "code": 200 - }, - "headers": { - "content-type": [ - "application/json; charset=utf-8" - ] - }, - "body": { - "string": "{\n \"note\": \"This is the API entry point - use a '*_api_url' link in 'meta' to search a collection.\",\n \"info\": {\n \"databaseName\": \"popit_django_test\",\n \"version\": \"0.0.14\" },\n \"meta\": {\n \"persons_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/persons\",\n \"organizations_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/organizations\",\n \"memberships_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/memberships\",\n \"posts_api_url\": \"http://popit-django-test.127.0.0.1.xip.io:3000/api/posts\"\n }\n }" - } - } - }, - { - "request": { - "body": null, - "headers": { - "content-type": [ - "application/json" + "Connection": [ + "keep-alive" ], "Accept-Encoding": [ "gzip, deflate" ], - "accept": [ - "application/json" + "Accept": [ + "*/*" ], "User-Agent": [ - "python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-49-generic" + "python-requests/2.10.0" ] }, "method": "GET", - "uri": "http://popit-django-test.127.0.0.1.xip.io:3000/api/persons/?per_page=50&page=1" + "uri": "http://whatever.127.0.0.1.xip.io:3000/api/v0.1/export.json" }, "response": { "status": { @@ -63,28 +28,19 @@ }, "headers": { "content-length": [ - "937" - ], - "x-content-type-options": [ - "nosniff" + "606" ], "x-powered-by": [ "Express" ], - "set-cookie": [ - "connect.sess=s%3Aj%3A%7B%22passport%22%3A%7B%7D%7D.x0Zr68Ve2M6fQjuaaiR0w0ShFrSKrrunIzT%2Fh19jUbM; Domain=.127.0.0.1.xip.io; Path=/; HttpOnly" - ], - "vary": [ - "X-HTTP-Method-Override, Origin" - ], "connection": [ - "close" + "keep-alive" ], "etag": [ - "W/\"3a9-449682be\"" + "W/\"25e-1736e4de\"" ], "date": [ - "Mon, 24 Aug 2015 14:45:25 GMT" + "Thu, 25 Aug 2016 13:19:29 GMT" ], "access-control-allow-origin": [ "*" @@ -94,7 +50,7 @@ ] }, "body": { - "string": "{\n \"total\": 2,\n \"page\": 1,\n \"per_page\": 30,\n \"has_more\": false,\n \"result\": [\n {\n \"html_url\": \"http://fiera.127.0.0.1.xip.io:3000/persons/raton-inteligente\",\n \"url\": \"http://fiera.127.0.0.1.xip.io:3000/api/v0.1/persons/raton-inteligente\",\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"html_url\": \"http://fiera.127.0.0.1.xip.io:3000/persons/fiera-feroz\",\n \"url\": \"http://fiera.127.0.0.1.xip.io:3000/api/v0.1/persons/fiera-feroz\",\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"email\": null,\n \"images\": [],\n \"memberships\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ]\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra buena onda\",\n \"email\": null,\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n }\n ],\n \"organizations\": [],\n \"memberships\": [],\n \"posts\": []\n}" } } } From 87f4fbebde74c75d1495fa5313309d61d96cc92d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:07:43 +0100 Subject: [PATCH 31/51] Use old_popit_url instead of popit_url in the API code & tests --- nuntium/api.py | 5 +++-- nuntium/tests/api/message_resource_test.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nuntium/api.py b/nuntium/api.py index 9fb1b4ec..8f397448 100644 --- a/nuntium/api.py +++ b/nuntium/api.py @@ -38,7 +38,7 @@ class Meta: authentication = ApiKeyAuthentication() def dehydrate(self, bundle): - bundle.data['resource_uri'] = bundle.obj.popit_url + bundle.data['resource_uri'] = bundle.obj.old_popit_url return bundle @@ -236,7 +236,8 @@ def dehydrate(self, bundle): if 'author_email' in bundle.data: bundle.data.pop('author_email', None) for person in bundle.obj.people: - bundle.data['persons'].append(person.popit_url) + # FIXME: + bundle.data['persons'].append(person.old_popit_url) return bundle def obj_create(self, bundle, **kwargs): diff --git a/nuntium/tests/api/message_resource_test.py b/nuntium/tests/api/message_resource_test.py index 5e13f16d..c8876e5d 100644 --- a/nuntium/tests/api/message_resource_test.py +++ b/nuntium/tests/api/message_resource_test.py @@ -133,7 +133,7 @@ def test_the_message_has_the_people_it_was_sent_to(self): message_from_the_api = messages[0] message = Message.objects.get(id=messages[0]['id']) for person in message_from_the_api['people']: - self.assertIn('popit_url', person) + self.assertIn('old_popit_url', person) self.assertIn( PopoloPerson.objects.get(id=person['id']), @@ -148,7 +148,7 @@ def test_create_a_new_message(self): 'subject': 'new message', 'content': 'the content thing', 'writeitinstance': '/api/v1/instance/{0}/'.format(writeitinstance.id), - 'persons': [writeitinstance.persons.all()[0].popit_url], + 'persons': [writeitinstance.persons.all()[0].old_popit_url], } url = '/api/v1/message/' @@ -179,7 +179,7 @@ def test_create_a_new_message_in_not_my_instance(self): 'subject': 'new message', 'content': 'the content thing', 'writeitinstance': '/api/v1/instance/{0}/'.format(writeitinstance.id), - 'persons': [person1.popit_url], + 'persons': [person1.old_popit_url], } url = '/api/v1/message/' @@ -194,7 +194,7 @@ def test_create_a_new_message_with_a_non_existing_person(self): 'content': 'the content thing', 'writeitinstance': '/api/v1/instance/{0}/'.format(writeitinstance.id), 'persons': [ - writeitinstance.persons.all()[0].popit_url, + writeitinstance.persons.all()[0].old_popit_url, 'http://this.person.does.not.exist', ], } @@ -213,7 +213,7 @@ def test_create_a_new_message_confirmated(self): 'subject': 'new message', 'content': 'the content thing', 'writeitinstance': '/api/v1/instance/{0}/'.format(writeitinstance.id), - 'persons': [writeitinstance.persons.all()[0].popit_url], + 'persons': [writeitinstance.persons.all()[0].old_popit_url], } url = '/api/v1/message/' self.api_client.post(url, data=message_data, format='json', authentication=self.get_credentials()) From 835e0e7465b46f643b28bf3a18e6bf74b5919058 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:11:29 +0100 Subject: [PATCH 32/51] Switch a test from using PopitApiInstance to PopoloSource --- nuntium/tests/api/instance_resource_test.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index 0bc19b03..3473466c 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from django.core.management import call_command from instance.models import WriteItInstance, PopoloPerson +from popolo_sources.models import PopoloSource from ...models import Message, Confirmation from tastypie.test import ResourceTestCase, TestApiClient from django.contrib.auth.models import User @@ -217,14 +218,17 @@ def test_filter_by_persons_popit_id(self): self.assertEquals(len(messages), 1) self.assertEquals(messages[0]['id'], self.message1.id) - def test_create_a_message_with_a_person_that_is_in_2_api_instances(self): - api = PopitApiInstance.objects.create(id=3, - url="https://popit.org/api/v1") - p = PopoloPerson.objects.create(api_instance=api, - name="Otro", - popit_url="https://popit.org/api/v1/persons/1", - popit_id="52bc7asdasd34567" - ) + def test_create_a_message_with_a_person_that_is_in_2_popolo_sources(self): + popolo_source = PopoloSource.objects.create( + id=3, + url="https://popit.org/api/v1" + ) + p = PopoloPerson.objects.create( + name="Otro", + popit_url="https://popit.org/api/v1/persons/1", + popit_id="52bc7asdasd34567" + ) + popolo_source.persons.add(p) url = '/api/v1/instance/%(writeitinstance_id)i/messages/' % { 'writeitinstance_id': self.writeitinstance.id, } From f5f5c3ca18346ffe4417054e06d7f9aed590511a Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 09:12:51 +0100 Subject: [PATCH 33/51] Update the MessagePerPersonView to use popolo_source_id --- nuntium/tests/messages_per_person_view_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuntium/tests/messages_per_person_view_test.py b/nuntium/tests/messages_per_person_view_test.py index 3c456a62..91ae2158 100644 --- a/nuntium/tests/messages_per_person_view_test.py +++ b/nuntium/tests/messages_per_person_view_test.py @@ -53,7 +53,7 @@ def test_person_id_param(self): 'messages_per_person_id', subdomain=self.writeitinstance.slug, kwargs={ - 'person_id': self.pedro.popit_id, + 'person_id': self.pedro.id_in_popolo_source, } ) response = self.client.get(url) From 6033f39c3ca6af31b6aacd4603a57e0c25d28118 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 25 Aug 2016 13:30:16 +0100 Subject: [PATCH 34/51] Fixes for the API tests --- instance/models.py | 14 ++++++++++++++ nuntium/api.py | 21 ++++++++++++++------- nuntium/models.py | 3 ++- nuntium/tests/api/answer_resource_test.py | 8 ++++---- nuntium/tests/api/instance_resource_test.py | 18 ++++++++++-------- nuntium/tests/api/message_resource_test.py | 2 +- 6 files changed, 45 insertions(+), 21 deletions(-) diff --git a/instance/models.py b/instance/models.py index f6e4ce50..b78e6a40 100644 --- a/instance/models.py +++ b/instance/models.py @@ -25,6 +25,19 @@ from mailit import MailChannel +class PopoloPersonQuerySet(models.QuerySet): + + def get_from_api_uri(self, uri_from_api): + try: + return PopoloPerson.objects.get( + identifiers__scheme='popit_url', + identifiers__identifier=uri_from_api) + except PopoloPerson.DoesNotExist: + return PopoloPerson.objects.get( + identifiers__scheme='popolo_uri', + identifiers__identifier=uri_from_api) + + class PopoloPerson(Person): class Meta: proxy = True @@ -109,6 +122,7 @@ def uri_for_api(self): msg = "Could find no global identifier for PopoloPerson with ID {0}" raise PopoloIdentifier.DoesNotExist(msg.format(self.pk)) + def today_in_date_range(start_date, end_date): today = str(datetime.date.today()) is_valid = ((not start_date) or str(start_date) <= today) and \ diff --git a/nuntium/api.py b/nuntium/api.py index 8f397448..11c9e9be 100644 --- a/nuntium/api.py +++ b/nuntium/api.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- + +import json + from tastypie.resources import ModelResource, ALL_WITH_RELATIONS, Resource from instance.models import PopoloPerson, WriteItInstance from .models import Message, Answer, \ @@ -38,7 +41,9 @@ class Meta: authentication = ApiKeyAuthentication() def dehydrate(self, bundle): - bundle.data['resource_uri'] = bundle.obj.old_popit_url + bundle.data['resource_uri'] = bundle.obj.uri_for_api() + bundle.data['popit_id'] = bundle.obj.old_popit_id + bundle.data['popit_url'] = bundle.obj.uri_for_api() return bundle @@ -92,7 +97,7 @@ def dehydrate(self, bundle): def add_persons_to_bundle(self, bundle): bundle.data['persons'] = [] for person in bundle.obj.persons.all(): - bundle.data['persons'].append(person.popit_url) + bundle.data['persons'].append(person.uri_for_api()) return bundle def hydrate(self, bundle): @@ -195,7 +200,10 @@ def build_filters(self, filters=None, ignore_bad_filters=False): raise Http404("PopoloPerson does not exist") if 'person__popit_id' in filters: try: - person = queryset.get(popit_id=filters['person__popit_id']) + person = queryset.get( + identifiers__scheme='popit_id', + identifiers__identifier=filters['person__popit_id'], + ) except ObjectDoesNotExist: raise Http404("PopoloPerson does not exist") if person: @@ -221,9 +229,9 @@ def hydrate(self, bundle): for person in instance.persons.all(): persons.append(person) else: - for popit_url in bundle.data['persons']: + for person_api_url in bundle.data['persons']: try: - person = PopoloPerson.objects.get(popit_url=popit_url) + person = PopoloPerson.objects.get_from_api_uri(person_api_url) persons.append(person) except ObjectDoesNotExist: pass @@ -236,8 +244,7 @@ def dehydrate(self, bundle): if 'author_email' in bundle.data: bundle.data.pop('author_email', None) for person in bundle.obj.people: - # FIXME: - bundle.data['persons'].append(person.old_popit_url) + bundle.data['persons'].append(person.uri_for_api()) return bundle def obj_create(self, bundle, **kwargs): diff --git a/nuntium/models.py b/nuntium/models.py index da63ead7..fd2fd529 100644 --- a/nuntium/models.py +++ b/nuntium/models.py @@ -545,8 +545,9 @@ def create_answer(cls, identifier_key, content, content_html=""): identifier = cls.objects.get(key=identifier_key) message = identifier.outbound_message.message person = identifier.outbound_message.contact.person + popolo_person = PopoloPerson.create_from_base_instance(person) the_created_answer = Answer.objects.create(message=message, - person=person, + person=popolo_person, content=content, content_html=content_html) return the_created_answer diff --git a/nuntium/tests/api/answer_resource_test.py b/nuntium/tests/api/answer_resource_test.py index 6852689e..f2e2194c 100644 --- a/nuntium/tests/api/answer_resource_test.py +++ b/nuntium/tests/api/answer_resource_test.py @@ -49,7 +49,7 @@ def test_get_the_list_of_answers_per_instance(self): def test_get_only_answers_of_the_instance(self): """Show answers from the writeitinstance only""" the_other_instance = WriteItInstance.objects.get(id=2) - person = Person.objects.get(id=1) + person = PopoloPerson.objects.get(id=1) message = the_other_instance.message_set.all()[0] answer = Answer.objects.create( message=message, @@ -85,9 +85,9 @@ def test_get_the_person_that_answered(self): self.assertEquals(person["id"], db_answer.person.id) self.assertEquals(person["image"], db_answer.person.image) self.assertEquals(person["name"], db_answer.person.name) - self.assertEquals(person["popit_id"], db_answer.person.popit_id) - self.assertEquals(person["popit_url"], db_answer.person.popit_url) - self.assertEquals(person["resource_uri"], db_answer.person.popit_url) + self.assertEquals(person["popit_id"], db_answer.person.old_popit_id) + self.assertEquals(person["popit_url"], db_answer.person.uri_for_api()) + self.assertEquals(person["resource_uri"], db_answer.person.uri_for_api()) self.assertEquals(person["summary"], db_answer.person.summary) def test_answer_ordering(self): diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index 3473466c..0745b53d 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -60,7 +60,7 @@ def test_get_persons_of_an_instance(self): instance = self.deserialize(response) self.assertIn('persons', instance) pedro = PopoloPerson.objects.get(id=1) - self.assertIn(pedro.popit_url, instance['persons']) + self.assertIn(pedro.uri_for_api(), instance['persons']) def test_create_a_new_instance(self): instance_data = { @@ -210,7 +210,7 @@ def test_filter_by_persons_popit_id(self): 'writeitinstance_id': self.writeitinstance.id, } data = self.data - data['person__popit_id'] = self.pedro.popit_id + data['person__popit_id'] = self.pedro.old_popit_id response = self.api_client.get(url, data=data) self.assertValidJSONResponse(response) messages = self.deserialize(response)['objects'] @@ -223,17 +223,19 @@ def test_create_a_message_with_a_person_that_is_in_2_popolo_sources(self): id=3, url="https://popit.org/api/v1" ) - p = PopoloPerson.objects.create( - name="Otro", - popit_url="https://popit.org/api/v1/persons/1", - popit_id="52bc7asdasd34567" - ) + p = PopoloPerson.objects.create(name="Otro") + p.identifiers.create( + scheme='popit_url', + identifier="https://popit.org/api/v1/persons/1") + p.identifiers.create( + scheme='popit_id', + identifier='52bc7asdasd34567') popolo_source.persons.add(p) url = '/api/v1/instance/%(writeitinstance_id)i/messages/' % { 'writeitinstance_id': self.writeitinstance.id, } data = self.data - data['person__popit_id'] = p.popit_id + data['person__popit_id'] = p.old_popit_id response = self.api_client.get(url, data=data) self.assertValidJSONResponse(response) diff --git a/nuntium/tests/api/message_resource_test.py b/nuntium/tests/api/message_resource_test.py index c8876e5d..7f5da399 100644 --- a/nuntium/tests/api/message_resource_test.py +++ b/nuntium/tests/api/message_resource_test.py @@ -133,7 +133,7 @@ def test_the_message_has_the_people_it_was_sent_to(self): message_from_the_api = messages[0] message = Message.objects.get(id=messages[0]['id']) for person in message_from_the_api['people']: - self.assertIn('old_popit_url', person) + self.assertIn('popit_url', person) self.assertIn( PopoloPerson.objects.get(id=person['id']), From 813cc54f705e626f373231b164269dfe56a916d5 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 10:36:07 +0100 Subject: [PATCH 35/51] Update fixtures for current PopIt's ID handling --- nuntium/tests/fixtures/persons_with_memberships.js | 4 ++-- .../fixtures/vcr_cassettes/persons_with_memberships.yaml | 8 ++++---- writeit/settings.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nuntium/tests/fixtures/persons_with_memberships.js b/nuntium/tests/fixtures/persons_with_memberships.js index e8d33fa4..f86cc273 100644 --- a/nuntium/tests/fixtures/persons_with_memberships.js +++ b/nuntium/tests/fixtures/persons_with_memberships.js @@ -4,7 +4,7 @@ exports.organizations = { html_url: "https://", url: "https://", classification: "party", - id: "dogsfci", + _id: "dogsfci", name: "Dogs fci", memberships: [ { @@ -134,4 +134,4 @@ exports.memberships = { contact_details: [ ] } -}; \ No newline at end of file +}; diff --git a/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml b/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml index 3ab66064..e5bd8603 100644 --- a/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml +++ b/nuntium/tests/fixtures/vcr_cassettes/persons_with_memberships.yaml @@ -28,7 +28,7 @@ }, "headers": { "content-length": [ - "3153" + "3136" ], "x-powered-by": [ "Express" @@ -37,10 +37,10 @@ "keep-alive" ], "etag": [ - "W/\"CZfM3c0Tcl07j8n43iW97Q==\"" + "W/\"QUrVDsF97EiemmJ76ZfjnQ==\"" ], "date": [ - "Thu, 25 Aug 2016 13:30:32 GMT" + "Fri, 02 Sep 2016 09:08:35 GMT" ], "access-control-allow-origin": [ "*" @@ -50,7 +50,7 @@ ] }, "body": { - "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanoi.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra muy buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [\n {\n \"html_url\": \"https://\",\n \"url\": \"https://\",\n \"classification\": \"party\",\n \"id\": \"57bef28f7b68c342193dd2ea\",\n \"name\": \"Dogs fci\",\n \"images\": [],\n \"posts\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [\n {\n \"name\": \"Dogs\",\n \"id\": \"54e314e82ec9128d32084466\"\n }\n ]\n }\n ],\n \"memberships\": [\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"2011-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"2005-03-21\",\n \"end_date\": \"2009-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraNotBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2007-03-20\",\n \"person_id\": \"benito\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"benitoBoss\"\n },\n {\n \"html_url\": \"https://ratonNotBoss\",\n \"url\": \"https://ratonNotBoss\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2045-03-20\",\n \"person_id\": \"raton-inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"ratonNotBoss\"\n }\n ],\n \"posts\": []\n}" + "string": "{\n \"persons\": [\n {\n \"name\": \"Rat\u00f3n Inteligente\",\n \"email\": \"raton@ciudadanoi.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"raton-inteligente\"\n },\n {\n \"name\": \"Fiera Feroz\",\n \"summary\": \"Fiera es una perra muy buena onda\",\n \"email\": \"fiera@ciudadanointeligente.org\",\n \"image\": \"http://www.ciudadanointeligente.org/wp-content/uploads/2012/05/fiera.jpg\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"fiera@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"fiera-feroz\"\n },\n {\n \"name\": \"Benito\",\n \"summary\": \"Benito es el perro de la Cata\",\n \"image\": \"http://i.imgur.com/Vehnajb.jpg\",\n \"email\": \"benito@ciudadanointeligente.org\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [\n {\n \"id\": \"53b47e011fa1acd34505d9b3\",\n \"label\": \"email\",\n \"type\": \"email\",\n \"value\": \"benito@ciudadanointeligente.org\",\n \"note\": \"\"\n }\n ],\n \"identifiers\": [],\n \"other_names\": [],\n \"id\": \"benito\"\n }\n ],\n \"organizations\": [\n {\n \"html_url\": \"https://\",\n \"url\": \"https://\",\n \"classification\": \"party\",\n \"name\": \"Dogs fci\",\n \"images\": [],\n \"posts\": [],\n \"links\": [],\n \"contact_details\": [],\n \"identifiers\": [],\n \"other_names\": [\n {\n \"name\": \"Dogs\",\n \"id\": \"54e314e82ec9128d32084466\"\n }\n ],\n \"id\": \"dogsfci\"\n }\n ],\n \"memberships\": [\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"2011-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"2005-03-21\",\n \"end_date\": \"2009-03-21\",\n \"person_id\": \"fiera-feroz\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"fieraNotBoss\"\n },\n {\n \"html_url\": \"https://dogsfci\",\n \"url\": \"https://dogsfci\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"Boss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2007-03-20\",\n \"person_id\": \"benito\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"benitoBoss\"\n },\n {\n \"html_url\": \"https://ratonNotBoss\",\n \"url\": \"https://ratonNotBoss\",\n \"organization_id\": \"dogsfci\",\n \"role\": \"NotTheBoss\",\n \"start_date\": \"1987-03-21\",\n \"end_date\": \"2045-03-20\",\n \"person_id\": \"raton-inteligente\",\n \"images\": [],\n \"links\": [],\n \"contact_details\": [],\n \"id\": \"ratonNotBoss\"\n }\n ],\n \"posts\": []\n}" } } } diff --git a/writeit/settings.py b/writeit/settings.py index 7b70b83c..200bcff1 100644 --- a/writeit/settings.py +++ b/writeit/settings.py @@ -315,7 +315,7 @@ # POPIT TESTING RELATED TEST_POPIT_API_HOST_IP = '127.0.0.1' TEST_POPIT_API_PORT = '3000' -TEST_POPIT_API_SUBDOMAIN = 'popit-django-test' +TEST_POPIT_API_SUBDOMAIN = 'whatever' TEST_POPIT_API_URL = "http://%s.%s.xip.io:%s/api/v0.1/export.json" % ( TEST_POPIT_API_SUBDOMAIN, From 5bb137055d7f408a6137fab3825c2ede98c92228 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 10:50:49 +0100 Subject: [PATCH 36/51] Make (probably-should-be-deprecated) methods use the popit_id Identifier --- nuntium/views.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nuntium/views.py b/nuntium/views.py index 3700640c..58dc24a5 100644 --- a/nuntium/views.py +++ b/nuntium/views.py @@ -8,9 +8,8 @@ from haystack.views import SearchView from itertools import chain -from popolo.models import Person from django.db.models import Q -from instance.models import WriteItInstance +from instance.models import PopoloPerson, WriteItInstance from .models import Confirmation, Message, Moderation from .forms import MessageSearchForm, PerInstanceSearchForm @@ -92,8 +91,10 @@ def get(self, *args, **kwargs): if kwargs.get('step') == 'who' and 'person_id' in self.request.GET: person_id = self.request.GET['person_id'] try: - the_person = self.writeitinstance.persons.get(popit_id=person_id) - except Person.DoesNotExist: + the_person = self.writeitinstance.persons.get( + identifiers__scheme='popit_id', + identifiers__identifier=person_id) + except PopoloPerson.DoesNotExist: pass # If we've come to /who but there is only one person possible to @@ -231,8 +232,10 @@ def dispatch(self, request, *args, **kwargs): if 'pk' in self.kwargs: params = {'pk': self.kwargs['pk']} elif 'person_id' in self.kwargs: - params = {'popit_id': self.kwargs['person_id']} - self.person = get_object_or_404(Person, **params) + params = { + 'identifiers__scheme': 'popit_id', + 'identifiers__identifier': self.kwargs['person_id']} + self.person = get_object_or_404(PopoloPerson, **params) self.writeitinstance = get_object_or_404(WriteItInstance, slug=request.subdomain) return super(MessagesPerPersonView, self).dispatch(request, *args, **kwargs) From d40922b69329a721dbc5b4e536ee668ff6a93ae7 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 10:51:51 +0100 Subject: [PATCH 37/51] Change the resync URL's kwarg to use popolo_source_pk, not popit_api_pk --- nuntium/subdomain_urls.py | 2 +- .../writeitinstance_and_popit_relations.html | 2 +- nuntium/tests/popit_writeit_relation_tests.py | 14 +++++++------- nuntium/user_section/views.py | 4 +++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nuntium/subdomain_urls.py b/nuntium/subdomain_urls.py index 5f649d48..d1a51e48 100644 --- a/nuntium/subdomain_urls.py +++ b/nuntium/subdomain_urls.py @@ -69,7 +69,7 @@ url(r'^settings/webhooks/$', WriteItInstanceWebHooksView.as_view(), name='writeitinstance_webhooks'), url(r'^settings/create_webhooks/$', WriteItInstanceCreateWebHooksView.as_view(), name='writeitinstance_create_webhooks'), url(r'^settings/sources/$', WriteitPopitRelatingView.as_view(), name='relate-writeit-popit'), - url(r'^settings/sources/resync/(?P[-\d]+)/$', ReSyncFromPopit.as_view(), name='resync-from-popit'), + url(r'^settings/sources/resync/(?P[-\d]+)/$', ReSyncFromPopit.as_view(), name='resync-from-popit'), url(r'^settings/sources/update/(?P[-\d]+)/$', WriteItPopitUpdateView.as_view(), name='update-popit-writeit-relation'), url(r'^settings/templates/$', WriteItInstanceTemplateUpdateView.as_view(), name='writeitinstance_template_update'), url(r'^settings/templates/new_answer_notification/$', NewAnswerNotificationTemplateUpdateView.as_view(), name='edit_new_answer_notification_template'), diff --git a/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html b/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html index 148139dc..efbb2d5d 100644 --- a/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html +++ b/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html @@ -60,7 +60,7 @@

{% trans 'Polling interval' %}

- +
diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index 7d333a6a..fe72ad96 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -259,11 +259,11 @@ def test_post_to_the_url_for_manual_resync(self): # This is just a symbolism but it is to show how this popit api is empty self.assertFalse(self.popolo_source.persons.all()) url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': self.popolo_source.pk}) + 'popolo_source_pk': self.popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = self.owner - response = ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) + response = ReSyncFromPopit.as_view()(request, popolo_source_pk=self.popolo_source.pk) self.assertEquals(response.status_code, 200) # It should have been updated @@ -280,22 +280,22 @@ def test_doesnt_add_another_relation_w_p(self): self.writeitinstance.writeitinstancepopitinstancerecord_set.all()) url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': another_popolo_source.pk}) + 'popolo_source_pk': another_popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = self.owner with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=another_popolo_source.pk) + ReSyncFromPopit.as_view()(request, popolo_source_pk=another_popolo_source.pk) def test_post_has_to_be_the_owner_of_the_instance(self): '''Only the owner of an instance can resync''' url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ - 'popit_api_pk': self.popolo_source.pk}) + 'popolo_source_pk': self.popolo_source.pk}) request = self.request_factory.post(url) request.subdomain = self.writeitinstance.slug request.user = AnonymousUser() with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) + ReSyncFromPopit.as_view()(request, popolo_source_pk=self.popolo_source.pk) other_user = User.objects.create_user(username="other_user", password="s3cr3t0") @@ -303,7 +303,7 @@ def test_post_has_to_be_the_owner_of_the_instance(self): request.subdomain = self.writeitinstance.slug request.user = other_user with self.assertRaises(Http404): - ReSyncFromPopit.as_view()(request, popit_api_pk=self.popolo_source.pk) + ReSyncFromPopit.as_view()(request, popolo_source_pk=self.popolo_source.pk) from nuntium.user_section.views import WriteItPopitUpdateView diff --git a/nuntium/user_section/views.py b/nuntium/user_section/views.py index 67b8fa5b..f71530fc 100644 --- a/nuntium/user_section/views.py +++ b/nuntium/user_section/views.py @@ -524,7 +524,9 @@ def post(self, request, *args, **kwargs): popolo_sources_previously_related = PopoloSource.objects.filter( writeitinstancepopitinstancerecord__writeitinstance=writeitinstance) - popolo_source = get_object_or_404(popolo_sources_previously_related, pk=kwargs['popit_api_pk']) + popolo_source = get_object_or_404( + popolo_sources_previously_related, + pk=kwargs['popolo_source_pk']) pull_from_popolo_json.delay(writeitinstance, popolo_source) return HttpResponse() From f927637923b773c851ff6042911a46bac205d805 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 10:57:16 +0100 Subject: [PATCH 38/51] Update start_local_popit_api.bash use current PopIt The start_local_popit_api.bash script is designed to make it easy to set up a local version of PopIt for testing against; this is still useful for re-recording the fixtures (see the commit message for 967f842529 and https://gist.github.com/mhl/848f7980d256ca38ff603375be4f264e ). However, this script installed a very old version of PopIt - this commit updates it to use the most recent version. (A key problem with the old version of this script is that it sets up a version of PopIt without the export.json endpoints.) --- start_local_popit_api.bash | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/start_local_popit_api.bash b/start_local_popit_api.bash index b6b0db01..e77ecd30 100755 --- a/start_local_popit_api.bash +++ b/start_local_popit_api.bash @@ -13,9 +13,8 @@ set -e # just checkout the mysociety-deploy branch # http://stackoverflow.com/a/7349740/5349 export DIR=popit-api-for-testing -export BRANCH=mysociety-deploy +export BRANCH=master export REMOTE_REPO=https://github.com/mysociety/popit-api.git -export PORT=3000 if [ ! -e $DIR ]; then mkdir $DIR; fi cd $DIR; @@ -26,10 +25,9 @@ if [ ! -e done.txt ]; then git remote add -t $BRANCH -f origin $REMOTE_REPO; git checkout $BRANCH; - echo "{ \"serverPort\": $PORT }" > config/general.json + echo "{}" > config/general.json # install the required node modules - npm install pow-mongodb-fixtures --quiet npm install --quiet touch done.txt; @@ -37,7 +35,7 @@ fi # Run the server in the background. Send access logging to file. -node server.js > access.log & +node test-server.js > access.log & # give it a chance to start and then print out the url to it sleep 2 From 3a2972dd8e2b17fc2cb0df81b2ca9511c1738239 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 15:26:55 +0100 Subject: [PATCH 39/51] Which of two identical email addresses is used doesn't matter This doesn't seem to matter, and since there is no Identifier associated with a ContactDetail in django-popolo, it'd be awkward to preserve the ContactDetail identifiers. It doesn't seem to be actually used anyway. --- nuntium/tests/popit_api_instance_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nuntium/tests/popit_api_instance_tests.py b/nuntium/tests/popit_api_instance_tests.py index 9a205418..2b8f9110 100644 --- a/nuntium/tests/popit_api_instance_tests.py +++ b/nuntium/tests/popit_api_instance_tests.py @@ -113,9 +113,6 @@ def test_bug_506(self): fiera = self.instance.persons.filter(name="Fiera Feroz") contacts = Contact.objects.filter(person=fiera) self.assertEquals(contacts.count(), 1) - # I'm prefering the one with popit_id and stuff - the_contact = contacts[0] - self.assertTrue(the_contact.popit_identifier) @popit_load_data(fixture_name='persons_with_memberships') def test_if_memberships_are_no_longer_active(self): From 67b18692e9f51a4feb68257f0b84abb01303b572 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 2 Sep 2016 15:37:50 +0100 Subject: [PATCH 40/51] Remove Contact.popit_identifier, which isn't used for anything --- .../0006_remove_contact_popit_identifier.py | 18 +++++++++++++++ contactos/models.py | 1 - contactos/tests/contacts_tests.py | 23 ------------------- 3 files changed, 18 insertions(+), 24 deletions(-) create mode 100644 contactos/migrations/0006_remove_contact_popit_identifier.py diff --git a/contactos/migrations/0006_remove_contact_popit_identifier.py b/contactos/migrations/0006_remove_contact_popit_identifier.py new file mode 100644 index 00000000..ae5977b7 --- /dev/null +++ b/contactos/migrations/0006_remove_contact_popit_identifier.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('contactos', '0005_rename_migrated_fields'), + ] + + operations = [ + migrations.RemoveField( + model_name='contact', + name='popit_identifier', + ), + ] diff --git a/contactos/models.py b/contactos/models.py index e5092e9f..042f6594 100644 --- a/contactos/models.py +++ b/contactos/models.py @@ -26,7 +26,6 @@ class Contact(models.Model): is_bounced = models.BooleanField(default=False) owner = models.ForeignKey(User, related_name="contacts", null=True) writeitinstance = models.ForeignKey('instance.WriteItInstance', related_name="contacts", null=True) - popit_identifier = models.CharField(max_length=512, null=True) enabled = models.BooleanField(default=True) def __unicode__(self): diff --git a/contactos/tests/contacts_tests.py b/contactos/tests/contacts_tests.py index 39691091..dede1583 100644 --- a/contactos/tests/contacts_tests.py +++ b/contactos/tests/contacts_tests.py @@ -74,14 +74,12 @@ def test_create_contact(self): value='contact point', person=self.person, writeitinstance=writeitinstance, - popit_identifier='12345' ) self.assertTrue(contact1) self.assertFalse(contact1.is_bounced) self.assertEquals(contact1.contact_type, contact_type) self.assertEquals(contact1.value, 'contact point') self.assertEquals(contact1.person, self.person) - self.assertEquals(contact1.popit_identifier, '12345') self.assertTrue(contact1.enabled) def test_contact_with_writeitinstance(self): @@ -96,7 +94,6 @@ def test_contact_with_writeitinstance(self): value='contact point', person=self.person, writeitinstance=writeitinstance, - popit_identifier='12345' ) self.assertTrue(contact1) self.assertFalse(contact1.is_bounced) @@ -104,26 +101,6 @@ def test_contact_with_writeitinstance(self): self.assertEquals(contact1.value, 'contact point') self.assertEquals(contact1.person, self.person) self.assertEquals(contact1.writeitinstance, writeitinstance) - self.assertEquals(contact1.popit_identifier, '12345') - - def test_create_contact_without_popit_identifier(self): - '''Create a contact without any reference to popit''' - contact_type = ContactType.objects.create( - name='mental message', - label_name='mental address id', - ) - writeitinstance = WriteItInstance.objects.get(id=1) - contact1 = Contact.objects.create( - contact_type=contact_type, - value='contact point', - person=self.person, - writeitinstance=writeitinstance, - ) - self.assertTrue(contact1) - self.assertFalse(contact1.is_bounced) - self.assertEquals(contact1.contact_type, contact_type) - self.assertEquals(contact1.value, 'contact point') - self.assertEquals(contact1.person, self.person) def test_contacts_reverse_name(self): # Yeah I did another test just to say that I have one more From 5083572471253da8afd96f03cd57663f3d466d3d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 14:44:38 +0100 Subject: [PATCH 41/51] Update references to source URLs in the templates --- nuntium/templates/nuntium/profiles/status-bar.html | 2 +- .../nuntium/profiles/writeitinstance_and_popit_relations.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nuntium/templates/nuntium/profiles/status-bar.html b/nuntium/templates/nuntium/profiles/status-bar.html index 22444874..ba1f03b3 100644 --- a/nuntium/templates/nuntium/profiles/status-bar.html +++ b/nuntium/templates/nuntium/profiles/status-bar.html @@ -18,7 +18,7 @@ {% for rec in writeitinstance.writeitinstancepopitinstancerecord_set.all %} {% if rec.status == 'inprogress' %} diff --git a/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html b/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html index efbb2d5d..e61d9cfd 100644 --- a/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html +++ b/nuntium/templates/nuntium/profiles/writeitinstance_and_popit_relations.html @@ -29,7 +29,7 @@

{% trans "Data Sources" %}

{% for record in relations %}
  • -

    {{ record.popitapiinstance.url }}

    +

    {{ record.popolo_source.url }}

    {{ record.status }} {% if record.status_explanation %}({{ record.status_explanation }}){% endif %} — last updated {{ record.updated }} From f3dcc3eaab9d21dad48efd22c01ee4fc16378481 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Sep 2016 17:10:17 +0100 Subject: [PATCH 42/51] Use uri_for_api instead of popit_url in the API documentation --- nuntium/templates/nuntium/writeitinstance_api_docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuntium/templates/nuntium/writeitinstance_api_docs.html b/nuntium/templates/nuntium/writeitinstance_api_docs.html index cfd6be84..68cef900 100644 --- a/nuntium/templates/nuntium/writeitinstance_api_docs.html +++ b/nuntium/templates/nuntium/writeitinstance_api_docs.html @@ -18,7 +18,7 @@

    {% trans "Creating a message using the API" %}

        'writeitinstance': '/api/v1/instance/{{ writeitinstance.id }}/',
        'persons': [
    {% for person in writeitinstance.persons.all %} -         '{{ person.popit_url }}'{% if not forloop.last %},{% endif %} ## For {{ person }}
    +         '{{ person.uri_for_api }}'{% if not forloop.last %},{% endif %} ## For {{ person }}
    {% endfor %}     ],
        'author_name': '{% trans "Name of sender" %}',
    From b0a63e02b709566dce40da617b220b8f02d23fcd Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 21 Sep 2016 11:18:46 +0100 Subject: [PATCH 43/51] Fix tests broken by switching to popolo_sources.PopoloSource --- instance/models.py | 18 ++++++++++++++++++ .../back_fill_writeit_popit_records.py | 15 +++++++++++---- nuntium/tests/api/instance_resource_test.py | 2 +- nuntium/tests/confirmation_template_test.py | 4 ++-- nuntium/tests/messages_test.py | 4 ++-- nuntium/tests/popit_writeit_relation_tests.py | 9 +++++---- .../tests/popit_instance_update_tests.py | 4 ++-- 7 files changed, 41 insertions(+), 15 deletions(-) diff --git a/instance/models.py b/instance/models.py index b78e6a40..c63f9eaf 100644 --- a/instance/models.py +++ b/instance/models.py @@ -7,6 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.core import mail from django.db import models, transaction +from django.db.models import Prefetch from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ @@ -37,6 +38,13 @@ def get_from_api_uri(self, uri_from_api): identifiers__scheme='popolo_uri', identifiers__identifier=uri_from_api) + def origin(self, popolo_source): + return self \ + .prefetch_related(Prefetch( + 'links_to_popolo_sources', + LinkToPopoloSource.objects.select_related('popolo_source'))) \ + .filter( + links_to_popolo_sources__popolo_source=popolo_source) class PopoloPerson(Person): class Meta: @@ -122,6 +130,16 @@ def uri_for_api(self): msg = "Could find no global identifier for PopoloPerson with ID {0}" raise PopoloIdentifier.DoesNotExist(msg.format(self.pk)) + # This is actually only used in tests; associations with a + # PopoloSource should all be handled by the update code in + # multiple-django-popolo-sources. + def add_link_to_popolo_source(self, popolo_source): + ct = ContentType.objects.get_for_model(Person) + LinkToPopoloSource.objects.get_or_create( + popolo_source=popolo_source, + content_type=ct, + object_id=self.id) + def today_in_date_range(start_date, end_date): today = str(datetime.date.today()) diff --git a/nuntium/management/commands/back_fill_writeit_popit_records.py b/nuntium/management/commands/back_fill_writeit_popit_records.py index 0ba8ebed..6bb80bb4 100644 --- a/nuntium/management/commands/back_fill_writeit_popit_records.py +++ b/nuntium/management/commands/back_fill_writeit_popit_records.py @@ -1,6 +1,7 @@ +from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand from instance.models import WriteitInstancePopitInstanceRecord -from popolo_sources.models import PopoloSource +from popolo_sources.models import LinkToPopoloSource, PopoloSource from django.contrib.auth.models import User import logging @@ -10,9 +11,15 @@ class WPBackfillRecords(object): @classmethod def back_fill_popit_records(cls, writeitinstance): - persons_in_instance = writeitinstance.persons.all() - popolo_sources = PopoloSource.objects.filter(persons__in=persons_in_instance).distinct() - for popolo_source in popolo_sources: + person_ct = ContentType.objects.get(app_label='popolo', model='person') + person_ids_in_instance = writeitinstance.persons. \ + values_list('id', flat=True) + popolo_sources_ids = LinkToPopoloSource.objects.filter( + content_type=person_ct, + object_id__in=person_ids_in_instance, + ).values_list('popolo_source', flat=True).distinct() + for popolo_source in PopoloSource.objects.filter( + pk__in=popolo_sources_ids): record, created = WriteitInstancePopitInstanceRecord.objects.get_or_create( writeitinstance=writeitinstance, popolo_source=popolo_source) diff --git a/nuntium/tests/api/instance_resource_test.py b/nuntium/tests/api/instance_resource_test.py index 0745b53d..ed83c30a 100644 --- a/nuntium/tests/api/instance_resource_test.py +++ b/nuntium/tests/api/instance_resource_test.py @@ -230,7 +230,7 @@ def test_create_a_message_with_a_person_that_is_in_2_popolo_sources(self): p.identifiers.create( scheme='popit_id', identifier='52bc7asdasd34567') - popolo_source.persons.add(p) + p.add_link_to_popolo_source(popolo_source) url = '/api/v1/instance/%(writeitinstance_id)i/messages/' % { 'writeitinstance_id': self.writeitinstance.id, } diff --git a/nuntium/tests/confirmation_template_test.py b/nuntium/tests/confirmation_template_test.py index 92d2e6b7..99f0bd8f 100644 --- a/nuntium/tests/confirmation_template_test.py +++ b/nuntium/tests/confirmation_template_test.py @@ -133,7 +133,7 @@ def test_rendering_templates(self): ) popolo_source = PopoloSource.objects.get(pk=1) recipient = PopoloPerson.objects.create(name='Test Person') - popolo_source.persons.add(recipient) + recipient.add_link_to_popolo_source(popolo_source) contact_type = ContactType.objects.create( name='Contact Type Name', label_name='Contact Type Label', @@ -249,7 +249,7 @@ def setUp(self): ) popolo_source = PopoloSource.objects.get(pk=1) recipient = PopoloPerson.objects.create(name='Test Person') - popolo_source.persons.add(recipient) + recipient.add_link_to_popolo_source(popolo_source) contact_type = ContactType.objects.create( name='Contact Type Name', label_name='Contact Type Label', diff --git a/nuntium/tests/messages_test.py b/nuntium/tests/messages_test.py index d535aa36..731ae776 100644 --- a/nuntium/tests/messages_test.py +++ b/nuntium/tests/messages_test.py @@ -478,7 +478,7 @@ def setUp(self): owner=user, ) self.person1 = PopoloPerson.objects.create(name='Pedro') - popolo_source.persons.add(self.person1) + self.person1.add_link_to_popolo_source(popolo_source) # This test was a bug against mysql def test_a_message_with_a_changed_slug(self): @@ -542,7 +542,7 @@ def setUp(self): slug='instance-1', owner=user) self.person1 = PopoloPerson.objects.create(name='Pedro') - popolo_source.persons.add(self.person1) + self.person1.add_link_to_popolo_source(popolo_source) # This test was a bug against mysql def test_a_message_with_a_changed_slug(self): diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index fe72ad96..b6dc9cd2 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -1,7 +1,8 @@ # coding=utf-8 from global_test_case import GlobalTestCase as TestCase, popit_load_data from instance.models import ( - InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord) + InstanceMembership, WriteItInstance, WriteitInstancePopitInstanceRecord, + PopoloPerson) from popolo_sources.models import PopoloSource from django.utils.unittest import skip from django.contrib.auth.models import User @@ -233,7 +234,7 @@ def setUp(self): self.owner.save() self.popolo_source = PopoloSource.objects.first() # Empty the popit_api_instance - self.popolo_source.persons.all().delete() + PopoloPerson.objects.origin(self.popolo_source).delete() self.popolo_source.url = settings.TEST_POPIT_API_URL self.popolo_source.save() self.popit_writeit_record = WriteitInstancePopitInstanceRecord.objects.create( @@ -257,7 +258,7 @@ def setUp(self): def test_post_to_the_url_for_manual_resync(self): '''Resyncing can be done by posting to a url''' # This is just a symbolism but it is to show how this popit api is empty - self.assertFalse(self.popolo_source.persons.all()) + self.assertFalse(PopoloPerson.objects.origin(self.popolo_source)) url = reverse('resync-from-popit', subdomain=self.writeitinstance.slug, kwargs={ 'popolo_source_pk': self.popolo_source.pk}) request = self.request_factory.post(url) @@ -267,7 +268,7 @@ def test_post_to_the_url_for_manual_resync(self): self.assertEquals(response.status_code, 200) # It should have been updated - self.assertTrue(self.popolo_source.persons.all()) + self.assertTrue(PopoloPerson.objects.origin(self.popolo_source)) @popit_load_data() def test_doesnt_add_another_relation_w_p(self): diff --git a/nuntium/user_section/tests/popit_instance_update_tests.py b/nuntium/user_section/tests/popit_instance_update_tests.py index 9998bfaf..514cfb52 100644 --- a/nuntium/user_section/tests/popit_instance_update_tests.py +++ b/nuntium/user_section/tests/popit_instance_update_tests.py @@ -44,11 +44,11 @@ def test_update_creates_records_given_an_instance_2_persons(self): no matter if there are two persons related ''' w = WriteItInstance.objects.first() - popolo_source = w.persons.first().popolo_sources.first() + popolo_source = w.persons.first().popolo_source another_person = PopoloPerson.objects.create( name="Another Person but with the same Popolo source", ) - another_person.popolo_sources.add(popolo_source) + another_person.add_link_to_popolo_source(popolo_source) InstanceMembership.objects.create(writeitinstance=w, person=another_person) WPBackfillRecords.back_fill_popit_records(writeitinstance=w) records = WriteitInstancePopitInstanceRecord.objects.filter(writeitinstance=w) From 66160ecda2662fcee991b969cbd6489224e24b30 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 22 Sep 2016 09:04:45 +0100 Subject: [PATCH 44/51] Make a test docstring a bit more clear --- nuntium/tests/popit_writeit_relation_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuntium/tests/popit_writeit_relation_tests.py b/nuntium/tests/popit_writeit_relation_tests.py index b6dc9cd2..449aab2c 100644 --- a/nuntium/tests/popit_writeit_relation_tests.py +++ b/nuntium/tests/popit_writeit_relation_tests.py @@ -75,7 +75,7 @@ def test_it_does_not_try_to_replicate_the_memberships(self): @popit_load_data() def test_clean_memberships(self): - '''As part of bug #429 there can be several Membership between one writeitinstance and a person''' + '''As a result of #429 there might be several Membership between one writeitinstance and a person''' popolo_source, created = PopoloSource.objects.get_or_create(url=settings.TEST_POPIT_API_URL) writeitinstance = WriteItInstance.objects.create(name='instance 1', slug='instance-1', owner=self.owner) # there should be an amount of memberships From 1c63a1509fbb45bce508032a8ee27f6f2e6ef744 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 27 Oct 2016 17:18:52 +0100 Subject: [PATCH 45/51] Fix an intermittent test failure due to unpredictable ordering It's possible for the test: nuntium.tests.home_view_tests:HomeViewTestCase.test_list_instances ... to fail because the order that instances are listed is not specified. This commit orders the instances by their slug. --- nuntium/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuntium/views.py b/nuntium/views.py index 58dc24a5..bfadf9b5 100644 --- a/nuntium/views.py +++ b/nuntium/views.py @@ -19,7 +19,7 @@ class HomeTemplateView(TemplateView): def get_context_data(self, **kwargs): context = super(HomeTemplateView, self).get_context_data(**kwargs) - all_instances = WriteItInstance.objects.all() + all_instances = WriteItInstance.objects.order_by('slug') context['writeitinstances'] = all_instances return context From d579dabc0e70c5490d712b50823e8e82fdc150d8 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 27 Oct 2016 17:41:49 +0100 Subject: [PATCH 46/51] fixup! Fixes for the API tests --- instance/models.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/instance/models.py b/instance/models.py index c63f9eaf..479f1e1f 100644 --- a/instance/models.py +++ b/instance/models.py @@ -29,14 +29,9 @@ class PopoloPersonQuerySet(models.QuerySet): def get_from_api_uri(self, uri_from_api): - try: - return PopoloPerson.objects.get( - identifiers__scheme='popit_url', - identifiers__identifier=uri_from_api) - except PopoloPerson.DoesNotExist: - return PopoloPerson.objects.get( - identifiers__scheme='popolo_uri', - identifiers__identifier=uri_from_api) + return self.get( + identifiers__identifier=uri_from_api, + identifiers__scheme__in=['popit_url', 'popolo_uri']) def origin(self, popolo_source): return self \ From a91533c674fabaa196600b7a7bfb1754cff5b7d2 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 5 Dec 2016 08:22:46 +0000 Subject: [PATCH 47/51] If we can tell which people are current, only include them Since the migration to syncing people using django-popolo and multiple-django-popolo-sources, we're now syncing all people (even historic representatives) from EveryPolitician, not just those in the current term(s). However, we should only be allowing people to write to current representatives. The problem here is that the old behaviour was just to include everyone from the data source who had a contact and hadn't been deliberately disabled - some (or perhaps most) of these non-EveryPolitician data sources won't have any way to tell who is current or not; the assumption should continue that they should all be made available to write to. We could divide these cases into sources that look like they're from EveryPolitician or not from the URL, but that wouldn't help with people serving EP data from elsewhere, or after modification. Probably the best approach would be to only show current representatives for sources that have Event objects with classification "legislative period", and then decide who is current by looking at those who have memberships with a legislative_period_id corresponding to a current legislative period. Unfortunately, at the moment django-popolo does not include the Event class, and we need a workaround quickly for the Alpaca project, so this commit uses a different heuristic for when to attempt to filter people for being current memberships of a legislature: if the people from that source have any memberships at all, it only includes those people who have a current membership (of any organisation - of course this might be completely spurious). This is good enough for the Alpaca case, which will give us enough time to update django-popolo and multiple-django-popolo-sources so that we can do this properly for the main production writeinpublic site. This commit also doesn't address whether historic representatives are available via the API. --- instance/models.py | 4 ++++ nuntium/views.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/instance/models.py b/instance/models.py index 479f1e1f..4d1e5227 100644 --- a/instance/models.py +++ b/instance/models.py @@ -268,6 +268,10 @@ def add_person(self, person): def persons_with_contacts(self): return self.persons.filter(contact__writeitinstance=self, contact__isnull=False).distinct() + @property + def has_popolo_memberships(self): + return self.persons.filter(memberships__isnull=False).exists() + def relate_with_persons_from_popolo_json(self, popolo_source): try: importer = PopoloSourceImporter( diff --git a/nuntium/views.py b/nuntium/views.py index bfadf9b5..ebf9d8fc 100644 --- a/nuntium/views.py +++ b/nuntium/views.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import date, datetime from django.views.generic import TemplateView, DetailView, RedirectView, ListView from subdomains.utils import reverse From a08e7d5b86a3553d70768181fbae0a47f6a07d3d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 6 Jan 2017 12:56:32 +0000 Subject: [PATCH 48/51] Add a command to fix bad popolo_uri Identifier objects Part of the fix for https://github.com/mysociety/alpaca/issues/60 --- instance/management/__init__.py | 0 instance/management/commands/__init__.py | 0 .../instance_remove_extraneous_popolo_uri.py | 56 +++++++++ instance/tests/__init__.py | 0 instance/tests/test_remove_extraneous_ids.py | 109 ++++++++++++++++++ 5 files changed, 165 insertions(+) create mode 100644 instance/management/__init__.py create mode 100644 instance/management/commands/__init__.py create mode 100644 instance/management/commands/instance_remove_extraneous_popolo_uri.py create mode 100644 instance/tests/__init__.py create mode 100644 instance/tests/test_remove_extraneous_ids.py diff --git a/instance/management/__init__.py b/instance/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instance/management/commands/__init__.py b/instance/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instance/management/commands/instance_remove_extraneous_popolo_uri.py b/instance/management/commands/instance_remove_extraneous_popolo_uri.py new file mode 100644 index 00000000..90481426 --- /dev/null +++ b/instance/management/commands/instance_remove_extraneous_popolo_uri.py @@ -0,0 +1,56 @@ +from django.core.management.base import BaseCommand +from django.db import transaction + +from instance.models import PopoloPerson + +BAD_SUBSTRING = '#person-person/' + + +class Command(BaseCommand): + + help = "Remove any extraneous Identifier objects with scheme 'popolo_uri'" + + @transaction.atomic + def handle(self, *args, **options): + errors_found = False + for person in PopoloPerson.objects.all(): + qs = person.identifiers.filter(scheme='popolo_uri') + original_count = qs.count() + if original_count == 0: + msg = "There were no popolo_uri Identifier objects for person " \ + "{person} with ID {person_id}" + print msg.format(person=person, person_id=person.id) + errors_found = True + continue + if original_count == 1: + sole_identifier = qs.first() + if BAD_SUBSTRING in sole_identifier.identifier: + msg = "The only remaining popolo_uri Identifier for " \ + "person {person} with ID {person_id} was a " \ + "malformed legacy identifier: {bad_identifier}" + print msg.format( + person=person, + person_id=person.id, + bad_identifier=sole_identifier.identifier) + errors_found = True + continue + # Otherwise we have more than one identifier. Delete any + # of the bad legacy IDs: + qs.filter(identifier__contains=BAD_SUBSTRING).delete() + # Now all the remaining identifiers should be the same; if + # they're not, that's an error. + unique_remaining = set(qs.values_list('identifier', flat=True)) + if len(unique_remaining) > 1: + msg = "There were multiple conflicting IDs for person " \ + "{person} with ID {person_id}" + print msg.format(person=person, person_id=person.id) + for non_unique_id in sorted(unique_remaining): + print " ", non_unique_id + errors_found = True + continue + # Now remove all but one of the identifiers: + count = qs.count() + for i in qs[:count - 1]: + i.delete() + if errors_found: + raise Exception("Errors were found; not committing the transaction") diff --git a/instance/tests/__init__.py b/instance/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/instance/tests/test_remove_extraneous_ids.py b/instance/tests/test_remove_extraneous_ids.py new file mode 100644 index 00000000..cea7b765 --- /dev/null +++ b/instance/tests/test_remove_extraneous_ids.py @@ -0,0 +1,109 @@ +from contextlib import contextmanager +import re +import sys + +from django.core.management import call_command +from django.utils import six +from global_test_case import GlobalTestCase as TestCase + +from instance.models import PopoloPerson +from popolo.models import Identifier + + +@contextmanager +def capture_output(): + # Suggested here: http://stackoverflow.com/a/17981937/223092 + new_out, new_err = six.StringIO(), six.StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield new_out, new_err + finally: + sys.stdout, sys.stderr = old_out, old_err + + +class TestRemoveExtraneousIDs(TestCase): + + def setUp(self): + # Give everyone a popolo_uri Identifier: + self.total_people = PopoloPerson.objects.count() + for i, person in enumerate(PopoloPerson.objects.all()): + identifier = 'https://example.com/foo.json#person-{0}'.format(i) + person.identifiers.create( + scheme='popolo_uri', + identifier=identifier + ) + + def test_someone_with_no_popolo_uri(self): + person_with_missing_id = PopoloPerson.objects.get(name='Felipe') + person_with_missing_id.identifiers.filter( + scheme='popolo_uri').delete() + with capture_output() as (out, err): + with self.assertRaises(Exception): + call_command('instance_remove_extraneous_popolo_uri') + self.assertEqual( + out.getvalue(), + 'There were no popolo_uri Identifier objects for person Felipe with ID 3\n' + ) + # Check that all the identifiers are still present, + # i.e. nothing was deleted: + self.assertEqual( + Identifier.objects.filter(scheme='popolo_uri').count(), + self.total_people - 1) + + def test_someone_with_only_a_bad_legacy_id(self): + person_with_only_legacy_id = PopoloPerson.objects.get(name='Felipe') + person_with_only_legacy_id.identifiers.filter( + scheme='popolo_uri').delete() + person_with_only_legacy_id.identifiers.create( + scheme='popolo_uri', + identifier='https://example.com/foo.json#person-person/42') + with capture_output() as (out, err): + with self.assertRaises(Exception): + call_command('instance_remove_extraneous_popolo_uri') + self.assertEqual( + out.getvalue(), + 'The only remaining popolo_uri Identifier for person Felipe with ID 3 was a malformed legacy identifier: https://example.com/foo.json#person-person/42\n' + ) + # Check that all the identifiers are still present, + # i.e. nothing was deleted: + self.assertEqual( + Identifier.objects.filter(scheme='popolo_uri').count(), + self.total_people) + + def test_conflicting_ids(self): + person_with_only_legacy_id = PopoloPerson.objects.get(name='Felipe') + person_with_only_legacy_id.identifiers.create( + scheme='popolo_uri', + identifier='https://example.com/foo.json#person-abcde') + with capture_output() as (out, err): + with self.assertRaises(Exception): + call_command('instance_remove_extraneous_popolo_uri') + self.assertEqual( + out.getvalue(), + '''There were multiple conflicting IDs for person Felipe with ID 3 + https://example.com/foo.json#person-2 + https://example.com/foo.json#person-abcde\n''' + ) + # Check that all the identifiers are still present, + # i.e. nothing was deleted: + self.assertEqual( + Identifier.objects.filter(scheme='popolo_uri').count(), + self.total_people + 1) + + def test_normal_case(self): + # Give everyone duplicate identifiers and a legacy one: + for person in PopoloPerson.objects.all(): + identifier = person.identifiers.get(scheme='popolo_uri').identifier + for _ in range(2): + person.identifiers.create( + scheme='popolo_uri', identifier=identifier) + i = re.search(r'person-([0-9]+)', identifier).group(1) + fmt = 'https://example.com/foo.json#person-person/{0}' + bad_identifier = fmt.format(i) + person.identifiers.create( + scheme='popolo_uri', identifier=bad_identifier) + call_command('instance_remove_extraneous_popolo_uri') + self.assertEqual( + Identifier.objects.filter(scheme='popolo_uri').count(), + self.total_people) From 6f2713f041003e28099a9a2bb0e835f9ac7131f3 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 9 Jan 2017 11:08:39 +0000 Subject: [PATCH 49/51] Update the version of mysociety-django-popolo used to be 0.0.8 This makes the handling of a missing 'identifiers' array for people less confusing and adds the option to preserve particular IDs on syncing. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3963ceb5..1d987e25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ kombu==3.0.35 libsass==0.11.1 markdown2==2.3.1 mock==1.0.1 -mysociety-django-popolo==0.0.7 +mysociety-django-popolo==0.0.8 ndg-httpsclient==0.4.1 nose==1.3.7 oauthlib==1.1.2 From 51f010bb1a0a583c11c6af8ada8f7c31ae486995 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 9 Jan 2017 11:47:15 +0000 Subject: [PATCH 50/51] Preserve legacy identifiers when syncing from a data source The default behaviour is to update identifiers to remove any not in the upstream data source when syncing, but we've deliberately created some legacy IDs in migration that we want to keep. (n.b. the popolo:person scheme is used is implicitly preserved because we specified an id_prefix of 'popolo:') --- instance/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/instance/models.py b/instance/models.py index 4d1e5227..58f131a2 100644 --- a/instance/models.py +++ b/instance/models.py @@ -275,7 +275,13 @@ def has_popolo_memberships(self): def relate_with_persons_from_popolo_json(self, popolo_source): try: importer = PopoloSourceImporter( - popolo_source, id_prefix='popolo:', truncate='yes') + popolo_source, + id_prefix='popolo:', + truncate='yes', + id_schemes_to_preserve={ + 'person': {'popit_id', 'popit_url', + 'popit_django_person_id', 'popolo_uri'} + }) importer.add_observer(ExtraIdentifierCreator(popolo_source)) importer.add_observer(InstanceMembershipUpdater(self)) person_tracker = PersonTracker() From 3c708288c87628ebd56fd0e1e8edaf9fbfea4079 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 9 Jan 2017 12:26:45 +0000 Subject: [PATCH 51/51] Don't duplicate popolo_uri identifiers on resyncing data We were getting one new (identical) popolo_uri Identifier with each sync of the data; this commit should fix that. --- instance/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instance/models.py b/instance/models.py index 58f131a2..24cad6e4 100644 --- a/instance/models.py +++ b/instance/models.py @@ -189,9 +189,9 @@ def notify(self, collection, django_object, created, popolo_data): collection=collection, popolo_id=popolo_data['id'], ) - django_object.identifiers.create( + django_object.identifiers.update_or_create( scheme='popolo_uri', - identifier=uri) + defaults={'identifier': uri}) def notify_deleted(self, collection, django_object): pass