Skip to content

Commit 25c6052

Browse files
hmhrexhmhrex
committed
(#4) Basic logging.
1 parent 36708f9 commit 25c6052

File tree

3 files changed

+149
-59
lines changed

3 files changed

+149
-59
lines changed

‎comics/utils/comicimporter.py‎

Lines changed: 105 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import json, os, datetime, re, requests, requests_cache
1+
import json, os, datetime, re, requests, requests_cache, logging
22
from urllib.request import urlretrieve
33
from urllib.parse import quote_plus, unquote_plus
44
from comics.models import Arc, Character, Creator, Team, Publisher, Series, Issue, Roles, Settings
@@ -10,6 +10,9 @@
1010
class ComicImporter(object):
1111

1212
def __init__(self):
13+
# Setup logging
14+
self.logger = logging.getLogger('tenma')
15+
1316
# Setup requests caching
1417
requests_cache.install_cache('./media/CACHE/comicvine-cache', expire_after=1800)
1518
requests_cache.core.remove_expired_responses()
@@ -59,10 +62,16 @@ def _process_dir(self, path, excluded):
5962
cvid = self._find_issue_match(entry.name)
6063
if cvid != '':
6164
# Process issue with ComicVine
62-
self._process_issue(entry.path, cvid)
65+
try:
66+
self._process_issue(entry.path, cvid)
67+
except Exception:
68+
self.logger.exception('An error occurred while processing %s' % entry.path)
6369
else:
6470
# Process issue without ComicVine
65-
self._process_issue_without_cvid(entry.path)
71+
try:
72+
self._process_issue_without_cvid(entry.path)
73+
except Exception:
74+
self.logger.exception('An error occurred while processing %s' % entry.path)
6675
else:
6776
self._process_dir(entry.path, excluded)
6877

@@ -101,6 +110,7 @@ def _find_issue_match(self, filename):
101110
'''
102111

103112
# Initialize response
113+
found_issue = None
104114
cvid = ''
105115

106116
# Attempt to extract series name, issue number, and year
@@ -115,77 +125,100 @@ def _find_issue_match(self, filename):
115125

116126
if matching_series:
117127
if not matching_series[0].cvid == '':
118-
cvid = self._find_match_with_series(matching_series[0].cvid, issue_number)
119-
if not cvid == '':
120-
return cvid
121-
122-
# Query Parameters
123-
query_params = self.base_params
124-
query_params['resources'] = 'issue'
125-
query_params['field_list'] = self.query_issue_fields
126-
query_params['limit'] = self.query_issue_limit
127-
128-
# Check for series name and issue number, or just series name
129-
if series_name and issue_number:
130-
query_params['query'] = series_name + ' ' + issue_number
131-
query_response = requests.get(
132-
self.baseurl + 'search',
133-
params=query_params,
134-
headers=self.headers
135-
).json()
136-
elif series_name:
137-
query_params['query'] = series_name_url
138-
query_response = requests.get(
139-
self.baseurl + 'search',
140-
params=query_params,
141-
headers=self.headers
142-
).json()
128+
found_issue = self._find_match_with_series(matching_series[0].cvid, issue_number)
143129

144-
best_option_list = []
145-
146-
# Try to find the closest match.
147-
for issue in query_response['results']:
148-
item_year = datetime.date.today().year
149-
item_number = 1
150-
item_name = ''
151-
152-
if 'cover_date' in issue:
153-
if issue['cover_date']:
154-
item_year = issue['cover_date'][0:4]
155-
if 'issue_number' in issue:
156-
if issue['issue_number']:
157-
item_number = issue['issue_number']
158-
if 'name' in issue['volume']:
159-
if issue['volume']['name']:
160-
item_name = issue['volume']['name']
161-
item_name = utils.remove_special_characters(item_name)
130+
if found_issue is None:
131+
# Query Parameters
132+
query_params = self.base_params
133+
query_params['resources'] = 'issue'
134+
query_params['field_list'] = self.query_issue_fields
135+
query_params['limit'] = self.query_issue_limit
162136

137+
# Check for series name and issue number, or just series name
163138
if series_name and issue_number:
164-
score = (fuzz.ratio(item_name.lower(), series_name.lower()) + fuzz.partial_ratio(item_name.lower(), series_name.lower())) / 2
165-
if score >= 90:
166-
if item_number == issue_number:
167-
if item_year == issue_year:
168-
best_option_list.insert(0, issue['id'])
169-
break
170-
best_option_list.insert(0, issue['id'])
139+
query_params['query'] = series_name + ' ' + issue_number
140+
query_response = requests.get(
141+
self.baseurl + 'search',
142+
params=query_params,
143+
headers=self.headers
144+
).json()
145+
elif series_name:
146+
query_params['query'] = series_name_url
147+
query_response = requests.get(
148+
self.baseurl + 'search',
149+
params=query_params,
150+
headers=self.headers
151+
).json()
152+
153+
best_option_list = []
154+
155+
# Try to find the closest match.
156+
for issue in query_response['results']:
157+
item_year = datetime.date.today().year
158+
item_number = 1
159+
item_name = ''
160+
161+
if 'cover_date' in issue:
162+
if issue['cover_date']:
163+
item_year = issue['cover_date'][0:4]
164+
if 'issue_number' in issue:
165+
if issue['issue_number']:
166+
item_number = issue['issue_number']
167+
if 'name' in issue['volume']:
168+
if issue['volume']['name']:
169+
item_name = issue['volume']['name']
170+
item_name = utils.remove_special_characters(item_name)
171+
172+
if series_name and issue_number:
173+
score = (fuzz.ratio(item_name.lower(), series_name.lower()) + fuzz.partial_ratio(item_name.lower(), series_name.lower())) / 2
174+
if score >= 90:
175+
if item_number == issue_number:
176+
if item_year == issue_year:
177+
best_option_list.insert(0, issue)
178+
break
179+
best_option_list.insert(0, issue)
180+
181+
found_issue = best_option_list[0] if best_option_list else None
182+
183+
cvid = found_issue['id'] if found_issue else ''
184+
185+
if found_issue is not None:
186+
if 'name' in found_issue['volume']:
187+
if found_issue['volume']['name']:
188+
series = found_issue['volume']['name']
189+
elif matching_series:
190+
series = matching_series[0].name
191+
if 'issue_number' in found_issue:
192+
if found_issue['issue_number']:
193+
number = found_issue['issue_number']
194+
else:
195+
number = ''
196+
self.logger.info('\"%(filename)s\" was matched on Comic Vine as \"%(series)s - #%(number)s\" (%(CVID)s)' % {
197+
'filename': filename,
198+
'series': series,
199+
'number': number,
200+
'CVID': cvid
201+
})
202+
else:
203+
self.logger.warning('No match was found for \"%s\" on Comic Vine.' % filename)
171204

172-
return best_option_list[0] if best_option_list else ''
205+
return cvid
173206

174207
#==================================================================================================
175208

176209
def _find_match_with_series(self, series_cvid, issue_number):
177210
'''
178211
Try to retrieve a match based on an existing series name.
179212
180-
Returns a ComicVine ID.
213+
Returns an issue from list.
181214
'''
182215

183-
issue_cvid = ''
216+
found_issue = None
184217

185218
if issue_number:
186219
# Query Parameters
187220
query_params = self.base_params
188-
query_params['field_list'] = 'issues'
221+
query_params['field_list'] = 'issues,name'
189222

190223
# Attempt to find issue based on extracted Series Name and Issue Number
191224
query_response = requests.get(
@@ -198,9 +231,9 @@ def _find_match_with_series(self, series_cvid, issue_number):
198231
for issue in query_response['results']['issues']:
199232
item_number = issue['issue_number'] if issue['issue_number'] else ''
200233
if item_number == issue_number:
201-
issue_cvid = issue['id']
234+
found_issue = issue
202235

203-
return issue_cvid
236+
return found_issue
204237

205238
#==================================================================================================
206239

@@ -245,6 +278,12 @@ def _process_issue_without_cvid(self, filepath):
245278
else:
246279
self._reprocess_issue_without_cvid(matching_issue[0].id)
247280

281+
self.logger.info('\"%(filename)s\" was processed successfully as \"%(series)s - #%(number)s\"' % {
282+
'filename': filename,
283+
'series': issue.series.name,
284+
'number': issue.number
285+
})
286+
248287
#==================================================================================================
249288

250289
def _reprocess_issue_without_cvid(self, issue_id):
@@ -369,6 +408,13 @@ def _process_issue(self, filename, cvid):
369408
else:
370409
issue.teams.add(self._update_team(matching_team[0].id, team['api_detail_url']))
371410

411+
self.logger.info('\"%(filename)s\" was processed successfully as \"%(series)s - #%(number)s\" (%(CVID)s)' % {
412+
'filename': filename,
413+
'series': series.name,
414+
'number': issue.number,
415+
'CVID': issue.cvid
416+
})
417+
372418
#==================================================================================================
373419

374420
def _get_object_data(self, response):

‎logs/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

‎tenma/settings.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,48 @@
2929

3030
ALLOWED_HOSTS = []
3131

32+
# LOGGING
33+
LOGGING = {
34+
'version': 1,
35+
'disable_existing_loggers': False,
36+
'formatters': {
37+
'verbose': {
38+
'format': '%(levelname)s | %(asctime)s | %(module)s | %(message)s',
39+
'datefmt': '%m/%d/%Y %I:%M:%S %p',
40+
},
41+
'simple': {
42+
'format': '%(levelname)s %(message)s'
43+
},
44+
},
45+
'filters': {
46+
'require_debug_false': {
47+
'()': 'django.utils.log.RequireDebugFalse',
48+
},
49+
},
50+
'handlers': {
51+
'file': {
52+
'level':'DEBUG',
53+
'class':'logging.handlers.RotatingFileHandler',
54+
'formatter': 'verbose',
55+
'filename': os.path.join(BASE_DIR, 'logs', 'tenma.log'),
56+
'maxBytes': 1024*1024*15, # 15MB
57+
'backupCount': 5,
58+
},
59+
},
60+
'loggers': {
61+
'django.server': {
62+
'handlers': ['file'],
63+
'level': 'DEBUG',
64+
'propagate': True,
65+
},
66+
'tenma': {
67+
'handlers': ['file'],
68+
'level': 'DEBUG',
69+
'propagate': True,
70+
},
71+
}
72+
}
73+
3274
# Application definition
3375

3476
INSTALLED_APPS = [

0 commit comments

Comments
 (0)