Ben's Working Branch

README.md

@@ -19,8 +19,8 @@ Refer to the steps outlined in DEPLOY.md for information to help with the initia
 ## Installing the CPD project
 
 After you've installed Tarbell, you can install the CPD project with the following command:
-
-    tarbell install git@bitbucket.org:projectlargo/cpd-settlements.git
+    
+    tarbell install https://github.com/INN/cpd-settlements
 
 Tarbell will prompt you: "Where would you like to create this project? [/Users/yourusernamegoeshere/tarbell/cpd-settlements]"
 
@@ -39,7 +39,11 @@ This project uses npm (grunt) and bower for its pre-deployment build process and
     npm install
     bower install
 
-Once you've installed the npm and bower requirements, run the build process:
+Once you've installed the npm and bower requirements, install grunt if necessary:
+
+    npm install -g grunt-cli
+
+Then run the build process:
 
     grunt build
 

__init__.py

@@ -4,7 +4,7 @@
 from titlecase import titlecase
 
 from .inc.models import Case, Officer, Payment
-from .inc.helpers import format_currency, total_for_payments
+from .inc.helpers import format_currency, total_for_payments, total_for_payments_wrapper
 
 blueprint = Blueprint(
     __name__,
@@ -41,11 +41,13 @@ def search(init_view='cases'):
             primary_causes.append(case.primary_cause)
         except IndexError:
             pass
+        except AttributeError:
+            pass
 
     context.update({
         'init_view': init_view,
         'cases': sorted(
-            cases, key=lambda x: total_for_payments(x.payments, False), reverse=True),
+            cases, key=lambda x: total_for_payments_wrapper(x, False), reverse=True),
         'neighborhoods': sorted(
             neighborhoods.values(), key=lambda x: x.get('neighborhood')),
         'officers': Officer.objects,

get_data.py

@@ -13,15 +13,17 @@
     download_data()
     print "Prep officer data"
     prep_officer_data()
+    """
     print "Geocoding addresses..."
     geocode_addresses()
     try:
         print "Get boundaries for points..."
-        boundaries_for_points()
+        #boundaries_for_points()
     except:
         print "ERROR: Unable to get boundaries for points. Is the boundary service running?"
         print "Exiting..."
         exit()
+    """
     print "Rendering cases and officer front-end json..."
     render_cases_json()
     render_officers_json()

inc/base.py

@@ -79,8 +79,16 @@ def get_related(self, model=None, attribute='case_number'):
         """
         # Make sure both objects have the attribute we're looking for.
         # Raises an exception if one model is missing the attribute we're looking to match on.
-        getattr(self, attribute)
-        getattr(model.objects[0], attribute)
+        try:
+            getattr(self, attribute)
+            getattr(model.objects[0], attribute)
+        except IndexError:
+            if len(model.objects) == 0:
+                # awful hard to compare if there are no objects yet?
+                pass
+            else:
+                print( u'This is the attribute we look to match on: ' + attribute )
+                raise IndexError( u'One of the models is missing the attribute we\'re looking to match on.' )
 
         result = model.objects.filter(**{attribute: getattr(self, attribute)})
         return result

inc/helpers.py

@@ -138,11 +138,23 @@ def format_currency(amount):
 
 
 def total_for_payments(payments, format=True):
-    payouts = [p.payment for p in payments if p.payment is not None]
-    fees = [p.fees for p in payments if p.fees is not None]
-    total = sum(payouts + fees)
+    if payments:
+        payouts = [p.payment for p in payments if p.payment is not None]
+        fees = [p.fees for p in payments if p.fees is not None]
+        total = sum(payouts + fees)
+    else:
+        total = 0
 
     if format:
         return format_currency(total)
     else:
         return total
+
+def total_for_payments_wrapper(x, format=True):
+    """totals_for_payments wrapper with error handling if x.payments does not exist."""
+    debug = 0
+    try:
+        return total_for_payments(x.payments, format)
+    except AttributeError:
+        print("This case had no payments: " + x.case_number)
+        return 0

inc/models.py

@@ -45,11 +45,15 @@ class Case(BaseModel):
         'narrative': 'Narrative',
         # Tags
         'interaction_type': 'interaction_type',
-        'officers_tags': 'officers',
-        'victims_tags': 'victims',
+        #'officers_tags': 'officers_tags',
+        #'victims_tags': 'victims_tags',
         'misconduct_type': 'misconduct_type',
         'weapons_used': 'weapons_used',
         'outcome': 'outcome',
+        # mk
+        'latitude': 'Latitude',
+        'longitude': 'Longitude',
+        #'neighborhood': 'Neighborhood'
     }
 
     def get_slug(self):
@@ -59,11 +63,13 @@ def get_related_officers(self):
         result = []
 
         for officer in Officer.objects:
-            if self.case_number in officer.case_numbers:
-                result.append(officer)
+            if hasattr(officer, 'case_number'):
+                if self.case_number in officer.case_numbers:
+                    result.append(officer)
 
-        if len(result) < 2 and result[0].first == 'Unnamed' and result[0].last == 'Officers':
-            result = []
+        if len(result) < 2 and len(result) > 0:
+            if result[0].first == 'Unnamed' and result[0].last == 'Officers':
+                result = []
 
         return result
 
@@ -73,7 +79,7 @@ class Victims(BaseModel):
     type = 'victims'
 
     field_map = {
-        'timestamp': "Timestamp",
+        #'timestamp': "Timestamp",
         'case_number': "Case number",
         'victim_1': "Victim 1",
         'victim_1_race': "Victim 1 Race",
@@ -142,18 +148,26 @@ def get_slug(self):
     if filename in ['officers.json', 'cases.geocoded.boundaries.json']:
         model.objects = NonMappedModelList(filepath, model)
     else:
-        model.objects = ModelList(filepath, model, model.field_map)
+        try:
+            model.objects = ModelList(filepath, model, model.field_map)
+        except Exception,e:
+            import ipdb; ipdb.set_trace()
 
 # Do some heavy lifting up-front so this stuff gets stashed in memory
-for case in Case.objects:
-    case.payments = case.get_related(Payment)
-    case.officers = case.get_related_officers()
-    case.victims = case.get_related(Victims)
-    case.slug = case.get_slug()
-
-for officer in Officer.objects:
-    officer_payments = []
-    for case_no in officer.case_numbers:
-        officer_payments += Payment.objects.filter(case_number=case_no)
-    officer.total_payments = total_for_payments(officer_payments, False)
-    officer.slug = officer.get_slug()
+try:
+    for case in Case.objects:
+        case.payments = case.get_related(Payment)
+        case.officers = case.get_related_officers()
+        case.victims = case.get_related(Victims)
+        case.slug = case.get_slug()
+
+    for officer in Officer.objects:
+        officer_payments = []
+        for case_no in officer.case_numbers:
+            officer_payments += Payment.objects.filter(case_number=case_no)
+        officer.total_payments = total_for_payments(officer_payments, False)
+        officer.slug = officer.get_slug()
+except AttributeError:
+    """I think that this means that the initial data was empty"""
+    print( "If this is your first time running get_data.py, ignore this error." )
+    print( "If this is not, something has gone wrong in the case and officer objects." )

lib/download_data.py

@@ -7,15 +7,140 @@
 
 SPREADSHEETS = {
     # Cases
-    'cases': '1DyJz3uWjZXBFZIVpV1y0FxBElfA29U4NR41nX_5XT9w',
+    # https://docs.google.com/spreadsheets/d/1YnDgLxycT-jwjuxwBuuJjlFCXPk4u5P_3yA57gPWHgo/edit#gid=1235485359
+    # - id
+    # - CaseNumber
+    # - DateFiled
+    # - DateClosed
+    # - Judge
+    # - PlaintiffsLeadAttorney
+    # - PlaintiffsAttorneyLawFirm
+    # - CitysLeadAttorney
+    # - CitysAttorneyLawFirm
+    # - MagistrateJudge
+    # - DateofIncident
+    # - LocationListed
+    # - StreetAddress
+    # - City
+    # - State
+    # - EnteredBy
+    # - Fact-checkedby
+    # - Differences
+    # - Latitude
+    # - Longitude
+    # - CensusPlaceFips
+    # - CensusMsaFips
+    # - CensusMetDivFips
+    # - CensusMcdFips
+    # - CensusCbsaMicro
+    # - CensusCbsaFips
+    # - CensusBlock
+    # - CensusBlockGroup
+    # - CensusTract
+    # - CensusCountyFips
+    # - CensusStateFips
+    # - naaccrCertCode
+    # - MNumber
+    # - MPreDirectional
+    # - MName
+    # - MSuffix
+    # - MCity
+    # - MState
+    # - Neighborhood
+    # - Narrative
+    # - primary_cause
+    # - federal_causes
+    # - state_causes
+    # - interaction_type
+    # - officers
+    # - victims
+    # - misconduct_type
+    # - weapons_used
+    # - outcome
+    # - tags
+    #'cases': '1DyJz3uWjZXBFZIVpV1y0FxBElfA29U4NR41nX_5XT9w',
+    'cases': '1YnDgLxycT-jwjuxwBuuJjlFCXPk4u5P_3yA57gPWHgo',
+
     # Victims
-    'victims': '1bRYEa8tzryjbs8HpvDqnu15oHoLLlBO93GsAaIOrFvA',
+    # https://docs.google.com/spreadsheets/d/1_nUXS1jdWv_j5av368Reer3TBzQPBKnmvsBm2stKNsQ/edit#gid=0
+    # - Timestamp
+    # - Case number
+    # - Victim 1
+    # - Victim 1 Race
+    # - Victim 1 Age (If known)
+    # - Victim 1 Gender
+    # - Victim 1 Deceased
+    # - Victim 2
+    # - Victim 3
+    # - Victim 4
+    # - Victim 5
+    # - Victim 6
+    # - Victim 7
+    # - Victim 8
+    # - More Victims?
+    # - Notes/Flags
+    # - Entered By
+    # - Fact-checked by
+    #'victims': '1bRYEa8tzryjbs8HpvDqnu15oHoLLlBO93GsAaIOrFvA',
+    'victims': '1_nUXS1jdWv_j5av368Reer3TBzQPBKnmvsBm2stKNsQ',
+
     # Officers
-    'officers_full': '18TByyAf5TB4AiC8zNEkMWiOREHLAi6dDwzrLOGh5b14',
-    # casecops -- completes the Officers dataset
-    'casecops': '1r3G3mmJ9fQV3fOJt3zKDnpf5-jzm78GJErSiLm1_H8A',
-    # Payments
-    'payments': '1IC5B7qoVkCMmzq6abrTkokq2Ua6nBQFmo4-0m-mPVgQ',
+    # https://docs.google.com/spreadsheets/d/1Zs-3f0vrensScGo3lnnCf7xKrprric5M5HokgmENNHU/edit#gid=1038247715
+    # - id
+    # - last_name
+    # - first_name
+    # - middle_init
+    # - sex
+    # - race
+    # - dob_year
+    # - current_age
+    # - status
+    # - appointed_date
+    # - position_code
+    # - position_desc
+    # - cpd_unit
+    # - cpd_unit_desc
+    # - resignation_date
+    # - slug
+    # - notes
+    #'officers_full': '18TByyAf5TB4AiC8zNEkMWiOREHLAi6dDwzrLOGh5b14',
+    'officers_full': '1Zs-3f0vrensScGo3lnnCf7xKrprric5M5HokgmENNHU',
+
+    # casecops -- matches the officers dataset to the case dataset
+    # https://docs.google.com/spreadsheets/d/1dlR4egtqSVQpw01d3VFwrEHDDvv2Bg7Y7hVWzoZRzi8/edit#gid=131997155
+    # - id
+    # - case_no
+    # - slug
+    # - cop_first_name
+    # - cop_middle_initial
+    # - cop_last_name
+    # - badge_no
+    # - officer_atty
+    # - officer_atty_firm
+    # - entered_by
+    # - entered_when
+    # - fact_checked_by
+    # - fact_checked_when
+    # - matched_by
+    # - matched_when
+    # - note
+    # - flag
+    # - case_id
+    # - cop_id
+    #'casecops': '1r3G3mmJ9fQV3fOJt3zKDnpf5-jzm78GJErSiLm1_H8A',
+    'casecops': '1dlR4egtqSVQpw01d3VFwrEHDDvv2Bg7Y7hVWzoZRzi8',
+
+    # Payments: matches case to payments to payees.
+    # https://docs.google.com/spreadsheets/d/1gTcPfSEJ_qAZldr23BdW97NWRDa-kRSY1VZ3zf6JOGY/edit#gid=822728219
+    # - case_num
+    # - payee
+    # - payment
+    # - fees_costs
+    # - primary_case
+    # - disposition
+    # - date_paid
+    #'payments': '1IC5B7qoVkCMmzq6abrTkokq2Ua6nBQFmo4-0m-mPVgQ',
+    'payments': '1gTcPfSEJ_qAZldr23BdW97NWRDa-kRSY1VZ3zf6JOGY',
 }
 
 

lib/geocode_addresses.py

@@ -14,7 +14,7 @@ def main():
 def geocode_addresses():
     cases = ModelList('data/cases.json', Case, Case.field_map)
     cases_dicts = [case for case in cases]
-
+    """
     geocoder = GoogleV3(timeout=30)
 
     for case in cases_dicts:
@@ -30,7 +30,8 @@ def geocode_addresses():
             print "No address for case: %s" % case.get('case_number')
 
         time.sleep(1)
-
+    """
+    import ipdb; ipdb.set_trace()
     with open('data/cases.geocoded.json', 'wb') as f:
         f.write(json.dumps(cases_dicts))