' + \
'\n'.join(['' + \
cgi.escape(description) + ' ' \
for (doctype, description) \
@@ -841,6 +911,23 @@ def create_file_upload_interface(recid,
else:
restrictions_list = ''
+ copyrights_list += 'Same as record '
+ for copyright_item in copyright_items:
+ copyrights_list += ''+ copyright_item.get('key') + ' '
+ copyrights_list += '%(other)s ' % {'other': _("Other")}
+ copyrights_list += ' '
+
+ # Save the date of the record in JavaScript so it will accessible later
+ try:
+ record_date = get_record(recid).get('imprint','').get('date','')
+ except AttributeError:
+ record_date = ''
+ body += """""" % record_date
+
# List the files
body += '''
@@ -852,6 +939,8 @@ def create_file_upload_interface(recid,
body += create_file_row(bibdoc, can_delete_doctypes,
can_rename_doctypes,
can_revise_doctypes,
+ can_change_copyright_doctypes,
+ can_change_advanced_copyright_doctypes,
can_describe_doctypes,
can_comment_doctypes,
can_keep_doctypes,
@@ -859,21 +948,24 @@ def create_file_upload_interface(recid,
doctypes_list,
show_links,
can_restrict_doctypes,
+ copyright_items,
even=not (i % 2),
ln=ln,
form_url_params=form_url_params,
protect_hidden_files=protect_hidden_files)
body += ''
if len(cleaned_doctypes) > 0:
- (revise_panel, javascript_prefix) = javascript_display_revise_panel(action='add', target='', show_doctypes=True, show_keep_previous_versions=False, show_rename=can_name_new_files, show_description=True, show_comment=True, bibdocname='', description='', comment='', show_restrictions=True, restriction=len(restrictions_and_desc) > 0 and restrictions_and_desc[0][0] or '', doctypes=doctypes_list)
- body += '''%(javascript_prefix)s
'
@@ -885,8 +977,8 @@ def create_file_upload_interface(recid,
get_upload_file_interface_css() + \
body
- # Display markup of the revision panel. This one is also
- # printed only at the beginning, so that it does not need to
+ # Display markup of the revision and copyright panels. Those ones are also
+ # printed only at the beginning, so that they do not need to
# be returned with each response
body += revise_balloon % \
{'CFG_SITE_URL': CFG_SITE_URL,
@@ -894,6 +986,9 @@ def create_file_upload_interface(recid,
'filename_label': filename_label,
'description_label': description_label,
'comment_label': comment_label,
+ 'copyright_label': copyright_label,
+ 'copyrights': copyrights_list,
+ 'advanced_copyright_label': advanced_copyright_label,
'restrictions': restrictions_list,
'previous_versions_help': _('You can decide to hide or not previous version(s) of this file.').replace("'", "\\'"),
'revise_format_help': _('When you revise a file, the additional formats that you might have previously uploaded are removed, since they no longer up-to-date with the new file.').replace("'", "\\'"),
@@ -904,6 +999,10 @@ def create_file_upload_interface(recid,
'uploading_label': _('Uploading...'),
'postprocess_label': _('Please wait...'),
'submit_or_button': form_url_params and 'button' or 'submit'}
+ body += copyright_balloon % \
+ {
+ 'close': _('Close')
+ }
body += '''
'
if main_bibdocfile.get_type() in can_revise_doctypes or \
'*' in can_revise_doctypes and not (hidden_p and protect_hidden_files):
+ # Advanced copyright panel will be inside revise panel, so we need to
+ # prepare parameters for copyright here
+
+ # Copyright and license link
+ copyright_license = get_copyright_and_license(abstract_bibdoc['list_latest_files'][0])
+ # take the first bibdocfile from list_latest_files, because the
+ # copyright and license for every bibdocfile should be the same
+ # I don't think we need to check this condition
+ # if main_bibdocfile.get_type() in can_change_copyright_doctypes or \
+ # '*' in can_change_copyright_doctypes and not (hidden_p and protect_hidden_files):
+ copyright_holder = copyright_license.get('copyright_holder', '')
+ copyright_date = copyright_license.get('copyright_date', '')
+ copyright_message = copyright_license.get('copyright_message', '')
+ copyright_holder_contact = copyright_license.get('copyright_holder_contact', '')
+ license = copyright_license.get('license', '')
+ license_url = copyright_license.get('license_url', '')
+ license_body = copyright_license.get('license_body', '')
+
+ # Check which copyright/license option should be selected in select list
+ if copyright_holder or copyright_date or copyright_message or \
+ copyright_holder_contact or license or license_url or license_body:
+ # If at least one copyright/license information is provided it means that the copyright is not the same as record
+ for copyright in copyright_items:
+ copyright_value = json.loads(copyright.get('value')).get('Copyright')
+ license_value = json.loads(copyright.get('value')).get('License')
+ if copyright_value.get('Holder') == copyright_holder and \
+ copyright_value.get('Message') == copyright_message and \
+ copyright_value.get('Contact') == copyright_holder_contact and \
+ license_value.get('License') == license and \
+ license_value.get('Url') == license_url and \
+ license_value.get('Body') == license_body:
+ copyright = copyright.get('key')
+ break
+ else:
+ # None of the predefined value - someone manually set the copyrights
+ copyright = 'Other'
+ else:
+ # There were no copyrights for this file so far
+ copyright = ''
+
(revise_panel, javascript_prefix) = javascript_display_revise_panel(
action='revise',
target=abstract_bibdoc['get_docname'],
@@ -1047,9 +1198,19 @@ def create_file_row(abstract_bibdoc, can_delete_doctypes,
show_rename=(main_bibdocfile.get_type() in can_rename_doctypes) or '*' in can_rename_doctypes,
show_description=(main_bibdocfile.get_type() in can_describe_doctypes) or '*' in can_describe_doctypes,
show_comment=(main_bibdocfile.get_type() in can_comment_doctypes) or '*' in can_comment_doctypes,
+ show_copyright=(main_bibdocfile.get_type() in can_change_copyright_doctypes) or '*' in can_change_copyright_doctypes,
+ show_advanced_copyright=(main_bibdocfile.get_type() in can_change_advanced_copyright_doctypes) or '*' in can_change_advanced_copyright_doctypes,
bibdocname=abstract_bibdoc['get_docname'],
description=description,
comment=comment,
+ copyright=copyright,
+ copyright_holder=copyright_holder,
+ copyright_date=copyright_date,
+ copyright_message=copyright_message,
+ copyright_holder_contact=copyright_holder_contact,
+ license=license,
+ license_url=license_url,
+ license_body=license_body,
show_restrictions=(main_bibdocfile.get_type() in can_restrict_doctypes) or '*' in can_restrict_doctypes,
restriction=restriction,
doctypes=doctypes_list)
@@ -1106,9 +1267,19 @@ def create_file_row(abstract_bibdoc, can_delete_doctypes,
show_rename=False,
show_description=False,
show_comment=False,
+ show_copyright=False,
+ show_advanced_copyright=False,
bibdocname='',
description='',
comment='',
+ copyright='',
+ copyright_holder='',
+ copyright_date='',
+ copyright_message='',
+ copyright_holder_contact='',
+ license='',
+ license_url='',
+ license_body='',
show_restrictions=False,
restriction=restriction,
doctypes=doctypes_list)
@@ -1172,6 +1343,15 @@ def build_updated_files_list(bibdocs, actions, recid, display_hidden_files=False
for action, bibdoc_name, file_path, rename, description, \
comment, doctype, keep_previous_versions, \
file_restriction, create_related_formats in actions:
+ # Hack, rename parameters if the type of the action is copyrightChange
+ # so it's easier to track what is stored in which variable
+ # We can do this because each action has the same number of parameters
+ if action == "copyrightChange":
+ (copyright_holder, copyright_date,
+ copyright_message, copyright_holder_contact,
+ license, license_url, license_body) = (file_path,
+ rename, description, comment, doctype,
+ keep_previous_versions, file_restriction)
dirname, filename, fileformat = decompose_file(file_path)
i += 1
if action in ["add", "revise"] and \
@@ -1242,6 +1422,13 @@ def build_updated_files_list(bibdocs, actions, recid, display_hidden_files=False
docid=-1, status='',
checksum=checksum, more_info=more_info))
abstract_bibdocs[bibdoc_name]['updated'] = True
+ elif action == "copyrightChange":
+ copyright = pack_copyrights(copyright_holder, copyright_date,
+ copyright_message, copyright_holder_contact)
+ license = pack_license(license, license_url, license_body)
+ set_copyright_and_license(abstract_bibdocs[bibdoc_name]['list_latest_files'],
+ copyright, license)
+ abstract_bibdocs[bibdoc_name]['updated'] = True
# For each BibDoc for which we would like to create related
# formats, do build the list of formats that should be created
@@ -1291,6 +1478,10 @@ def log_action(log_dir, action, bibdoc_name, file_path, rename,
format was motivated by the need to have it easily readable by
other scripts. Not sure it still makes sense nowadays...
+ If we no longer need to escape the '---', maybe we can change those log
+ functions into something more generic like a log function
+ that records any arbitrary number of parameters ?
+
Newlines are also reserved, and are escaped from the input values
(necessary for the 'comment' field, which is the only one allowing
newlines from the browser)
@@ -1335,7 +1526,7 @@ def log_action(log_dir, action, bibdoc_name, file_path, rename,
try:
file_desc = open(log_file, "a+")
# We must escape new lines from comments in some way:
- comment = str(comment).replace('\\', '\\\\').replace('\r\n', '\\n\\r')
+ comment = str(comment).replace('\\', '\\\\').replace('\r\n', '\\n\\r').replace('\n', '\\n')
msg = action + '<--->' + \
bibdoc_name.replace('---', '___') + '<--->' + \
file_path + '<--->' + \
@@ -1351,10 +1542,64 @@ def log_action(log_dir, action, bibdoc_name, file_path, rename,
except Exception ,e:
raise e
+def log_copyright_action(log_dir, action, bibdoc_name, copyright_holder,
+ copyright_date, copyright_message,
+ copyright_holder_contact, license, license_url,
+ license_body):
+ """
+ Logs copyright change action in the actions log.
+ See log_action() function for more information
+ @param log_dir: directory where to save the log (ie. working_dir)
+
+ @param action: the performed action (one of 'revise', 'delete',
+ 'add', 'addFormat')
+
+ @param bibdoc_name: the name of the bibdoc on which the change is
+ applied
+
+ @param copyright_holder: holder of the copyright associated with the file
+
+ @param copyright_date: date of the copyright associated with the file
+
+ @param copyright_message: message associated with the copyright
+
+ @param copyright_holder_contact: contact information of the copyright holder
+
+ @param license: license of the file
+
+ @param license_url: url of the license related to the file
+
+ @param license_body: person or institution imposing the license
+ (author, publisher)
+
+ """
+ log_file = os.path.join(log_dir, 'bibdocactions.log')
+ try:
+ file_desc = open(log_file, "a+")
+ # We must escape new lines from copyright message in some way:
+ copyright_message = str(copyright_message).replace('\\', '\\\\').replace('\r\n', '\\n\\r').replace('\n', '\\n')
+ msg = action + '<--->' + \
+ bibdoc_name.replace('---', '___') + '<--->' + \
+ copyright_holder + '<--->' + \
+ copyright_date + '<--->' + \
+ copyright_message + '<--->' + \
+ copyright_holder_contact + '<--->' + \
+ license + '<--->' + \
+ license_url + '<--->' + \
+ license_body + '<--->' + \
+ '\n' # This last <---> is to match the number of parameters of log_action function
+ file_desc.write("%s --> %s" %(time.strftime("%Y-%m-%d %H:%M:%S"), msg))
+ file_desc.close()
+ except Exception, e:
+ raise e
+
+
def read_actions_log(log_dir):
"""
Reads the logs of action to be performed on files
+ Both log_copyright_action and log_action create rows with the same number
+ of parameters, so each action can be treated in the same way.
See log_action(..) for more information about the structure of the
log file.
@@ -1368,42 +1613,62 @@ def read_actions_log(log_dir):
file_desc = open(log_file, "r")
for line in file_desc.readlines():
(timestamp, action) = line.split(' --> ', 1)
- try:
- (action, bibdoc_name, file_path, rename, description,
- comment, doctype, keep_previous_versions,
- file_restriction, create_related_formats) = action.rstrip('\n').split('<--->')
- except ValueError, e:
- # Malformed action log
- pass
-
- # Clean newline-escaped comment:
- comment = comment.replace('\\n\\r', '\r\n').replace('\\\\', '\\')
-
- # Perform some checking
- if action not in CFG_ALLOWED_ACTIONS:
- # Malformed action log
- pass
-
- try:
- keep_previous_versions = int(keep_previous_versions)
- except:
- # Malformed action log
- keep_previous_versions = 1
- pass
-
- create_related_formats = create_related_formats == 'True' and True or False
-
- actions.append((action, bibdoc_name, file_path, rename, \
- description, comment, doctype,
- keep_previous_versions, file_restriction,
- create_related_formats))
+ if action.split('<--->', 1)[0] == 'copyrightChange':
+ try:
+ # if the action is "copyrightChange" we want to name
+ # parameters differently for clarity
+ (action, file_path, copyright_holder, copyright_date, copyright_message,
+ copyright_holder_contact, license, license_url,
+ license_body, _) = action.rstrip('\n').split('<--->')
+ except ValueError, e:
+ # Malformed action log
+ pass
+ # Clean newline-escaped copyright_message:
+ copyright_message = copyright_message.replace('\\n\\r', '\r\n').replace('\\\\', '\\').replace('\\n', '\n')
+ # Perform some checking
+ if action not in CFG_ALLOWED_ACTIONS:
+ # Malformed action log
+ pass
+ actions.append((action, file_path, copyright_holder, copyright_date,
+ copyright_message, copyright_holder_contact,
+ license, license_url, license_body, ''))
+ else:
+ try:
+ (action, bibdoc_name, file_path, rename, description,
+ comment, doctype, keep_previous_versions,
+ file_restriction, create_related_formats) = action.rstrip('\n').split('<--->')
+ except ValueError, e:
+ # Malformed action log
+ pass
+ # Clean newline-escaped comment:
+ comment = comment.replace('\\n\\r', '\r\n').replace('\\\\', '\\').replace('\\n', '\n')
+ try:
+ keep_previous_versions = int(keep_previous_versions)
+ except:
+ # Malformed action log
+ keep_previous_versions = 1
+ # Perform some checking
+ if action not in CFG_ALLOWED_ACTIONS:
+ # Malformed action log
+ pass
+ actions.append((action, bibdoc_name, file_path, rename,
+ description, comment, doctype,
+ keep_previous_versions, file_restriction,
+ create_related_formats))
file_desc.close()
except:
pass
return actions
-def javascript_display_revise_panel(action, target, show_doctypes, show_keep_previous_versions, show_rename, show_description, show_comment, bibdocname, description, comment, show_restrictions, restriction, doctypes):
+def javascript_display_revise_panel(action, target, show_doctypes,
+ show_keep_previous_versions, show_rename,
+ show_description, show_comment, show_copyright,
+ show_advanced_copyright, bibdocname, description,
+ comment, copyright, copyright_holder, copyright_date,
+ copyright_message, copyright_holder_contact,
+ license, license_url, license_body,
+ show_restrictions, restriction, doctypes):
"""
Returns a correctly encoded call to the javascript function to
display the revision panel.
@@ -1420,9 +1685,19 @@ def javascript_display_revise_panel(action, target, show_doctypes, show_keep_pre
"showRename": %(showRename)s,
"showDescription": %(showDescription)s,
"showComment": %(showComment)s,
+ "showCopyright": %(showCopyright)s,
+ "showAdvancedCopyright": %(showAdvancedCopyright)s,
"bibdocname": "%(bibdocname)s",
"description": "%(description)s",
"comment": "%(comment)s",
+ "copyright": "%(copyright)s",
+ "copyrightHolder": "%(copyrightHolder)s",
+ "copyrightDate": "%(copyrightDate)s",
+ "copyrightMessage": "%(copyrightMessage)s",
+ "copyrightHolderContact": "%(copyrightHolderContact)s",
+ "license": "%(license)s",
+ "licenseUrl": "%(licenseUrl)s",
+ "licenseBody": "%(licenseBody)s",
"showRestrictions": %(showRestrictions)s,
"restriction": "%(restriction)s",
"doctypes": "%(doctypes)s"}
@@ -1436,8 +1711,18 @@ def javascript_display_revise_panel(action, target, show_doctypes, show_keep_pre
'showKeepPreviousVersions': show_keep_previous_versions and 'true' or 'false',
'showComment': show_comment and 'true' or 'false',
'showDescription': show_description and 'true' or 'false',
+ 'showCopyright': show_copyright and 'true' or 'false',
+ 'showAdvancedCopyright': show_advanced_copyright and 'true' or 'false',
'description': description and escape_javascript_string(description, escape_for_html=False) or '',
'comment': comment and escape_javascript_string(comment, escape_for_html=False) or '',
+ 'copyright': copyright and escape_javascript_string(copyright, escape_for_html=False) or '',
+ 'copyrightHolder': copyright_holder and escape_javascript_string(copyright_holder, escape_for_html=False) or '',
+ 'copyrightDate': copyright_date and escape_javascript_string(copyright_date, escape_for_html=False) or '',
+ 'copyrightMessage': copyright_message and escape_javascript_string(copyright_message, escape_for_html=False) or '',
+ 'copyrightHolderContact': copyright_holder_contact and escape_javascript_string(copyright_holder_contact, escape_for_html=False) or '',
+ 'license': license and escape_javascript_string(license, escape_for_html=False) or '',
+ 'licenseUrl': license_url and escape_javascript_string(license_url, escape_for_html=False) or '',
+ 'licenseBody': license_body and escape_javascript_string(license_body, escape_for_html=False) or '',
'showRestrictions': show_restrictions and 'true' or 'false',
'restriction': escape_javascript_string(restriction, escape_for_html=False),
'doctypes': escape_javascript_string(doctypes, escape_for_html=False)}
@@ -1457,8 +1742,9 @@ def get_uploaded_files_for_docname(log_dir, docname):
"""
return [file_path for action, bibdoc_name, file_path, rename, \
description, comment, doctype, keep_previous_versions , \
- file_restriction, create_related_formats in read_actions_log(log_dir) \
- if bibdoc_name == docname and os.path.exists(file_path)]
+ file_restriction, copyright_license in read_actions_log(log_dir) \
+ if action in ['revise', 'add', 'addFormat'] and \
+ bibdoc_name == docname and os.path.exists(file_path)]
def get_bibdoc_for_docname(docname, abstract_bibdocs):
"""
@@ -1549,6 +1835,14 @@ def get_description_and_comment(bibdocfiles):
return (description, comment)
+def get_copyright_and_license(bibdocfile):
+ """
+ Returns the copyright and license of a BibDoc as a dictionary
+ """
+ copyright_license = bibdocfile.get_copyright()
+ copyright_license.update(bibdocfile.get_license())
+ return copyright_license
+
def set_description_and_comment(abstract_bibdocfiles, description, comment):
"""
Set the description and comment to the given (abstract)
@@ -1571,6 +1865,67 @@ def set_description_and_comment(abstract_bibdocfiles, description, comment):
bibdocfile.description = description
bibdocfile.comment = comment
+def set_copyright_and_license(abstract_bibdocfiles, copyright, license):
+ """
+ Set the copyright and license to the given (abstract)
+ bibdocfiles.
+
+ @param abstract_bibdocfiles: the list of 'abstract' files of a
+ given bibdoc for which we want to set the
+ copyright and license.
+
+ @param copyright: the new copyright
+ @param license: the new license
+ """
+ for bibdocfile in abstract_bibdocfiles:
+ bibdocfile.copyright = copyright
+ bibdocfile.license = license
+
+def pack_copyrights(copyright_holder, copyright_date, copyright_message,
+ copyright_holder_contact):
+ """
+ Packs the parameters into copyright dictionary with properly named keys.
+ Since we pass those copyright values in a few places, we pack them
+ here, so when the keys change, we won't have to rename them everywhere.
+
+ @param copyright_holder: holder of the copyright associated with the file
+ @type copyright_holder: string
+ @param copyright_date: date of the copyright associated with the file
+ @type copyright_date: string
+ @param copyright_message: message associated with the copyright
+ @type copyright_message: string
+ @param copyright_holder_contact: contact information of the copyright holder
+ @type copyright_holder_contact: string
+ """
+ copyright = {}
+ copyright['copyright_holder'] = copyright_holder
+ copyright['copyright_date'] = copyright_date
+ copyright['copyright_message'] = copyright_message
+ copyright['copyright_holder_contact'] = copyright_holder_contact
+
+ return copyright
+
+def pack_license(license, license_url, license_body):
+ """
+ Packs the parameters into license dictionary with properly named keys.
+ Since we pass those license values in a couple of places, we pack them
+ here, so when the keys change, we won't have to replace them everywhere
+
+ @param license: license of the file
+ @type license: string
+ @param license_url: url of the license related to the file
+ @type license_url: string
+ @param license_body: person or institution imposing the license
+ (author, publisher)
+ @type license_body: string
+ """
+ packed_license = {}
+ packed_license['license'] = license
+ packed_license['license_url'] = license_url
+ packed_license['license_body'] = license_body
+
+ return packed_license
+
def delete_file(working_dir, file_path):
"""
Deletes a file at given path from the file.
@@ -1641,7 +1996,9 @@ def wash_form_parameters(form, abstract_bibdocs, can_keep_doctypes,
@return: tuple (file_action, file_target, file_target_doctype,
keep_previous_files, file_description, file_comment,
- file_rename, file_doctype, file_restriction) where::
+ file_rename, file_doctype, file_restriction, copyright_holder,
+ copyright_date, copyright_message, copyright_holder_contact,
+ license, license_url, license_body) where::
file_action: *str* the performed action ('add',
'revise','addFormat' or 'delete')
@@ -1685,14 +2042,32 @@ def wash_form_parameters(form, abstract_bibdocs, can_keep_doctypes,
file_path: *str* the full path to the file
+ copyright_holder: *str* holder of the copyright of the file
+
+ copyright_date: *str* date of the copyright
+
+ copyright_message: *str* message of the copyright
+
+ copyright_holder_contact: *str* contact information of the copyright
+ holder
+
+ license: *str* license of the file
+
+ license_url: *str* url of the license related to the file
+
+ license_body: *str* person or institution imposing the license
+ (author, publisher)
+
@rtype: tuple(string, string, string, boolean, string, string,
- string, string, string, string, string)
+ string, string, string, string, string, string,
+ string, string, string, string, string, string)
"""
# Action performed ...
if form.has_key("fileAction") and \
form['fileAction'] in CFG_ALLOWED_ACTIONS:
file_action = str(form['fileAction']) # "add", "revise",
- # "addFormat" or "delete"
+ # "addFormat" "copyrightChange"
+ # or "delete"
else:
file_action = ""
@@ -1842,10 +2217,20 @@ def wash_form_parameters(form, abstract_bibdocs, can_keep_doctypes,
file_name = None
file_path = None
+ # Escape fields related to copyright and license
+ copyright_holder = str(form.get('copyrightHolder',''))
+ copyright_date = str(form.get('copyrightDate',''))
+ copyright_message = str(form.get('copyrightMessage',''))
+ copyright_holder_contact = str(form.get('copyrightHolderContact',''))
+ license = str(form.get('license',''))
+ license_url = str(form.get('licenseUrl',''))
+ license_body = str(form.get('licenseBody',''))
+
return (file_action, file_target, file_target_doctype,
keep_previous_files, file_description, file_comment,
file_rename, file_doctype, file_restriction, file_name,
- file_path)
+ file_path, copyright_holder, copyright_date, copyright_message,
+ copyright_holder_contact, license, license_url, license_body)
def move_uploaded_files_to_storage(working_dir, recid, icon_sizes,
@@ -1934,8 +2319,6 @@ def move_uploaded_files_to_storage(working_dir, recid, icon_sizes,
if new_bibdoc:
newly_added_bibdocs.append(new_bibdoc)
-
-
if create_related_formats:
# Schedule creation of related formats
create_related_formats_for_bibdocs[rename or bibdoc_name] = True
@@ -1947,13 +2330,22 @@ def move_uploaded_files_to_storage(working_dir, recid, icon_sizes,
elif action == 'delete':
delete(bibdoc_name, recid, working_dir, pending_bibdocs,
bibrecdocs)
+ elif action == 'copyrightChange':
+ # Don't worry about those strange variable names that are being
+ # send to those two functions below. Those variables store proper
+ # copyright and license data, there is just no point in renaming
+ # them only to pass them as arguments to the functions
+ copyright = pack_copyrights(file_path, rename, description, comment)
+ license = pack_license(doctype, keep_previous_versions, file_restriction)
+ copyright_license_change(file_path, bibdoc_name, copyright,
+ license, recid, working_dir, bibrecdocs)
# Finally rename bibdocs that should be named according to a file in
# curdir (eg. naming according to report number). Only consider
# file that have just been added.
parameters = _read_file_revision_interface_configuration_from_disk(working_dir)
new_names = []
- doctypes_to_default_filename = parameters[22]
+ doctypes_to_default_filename = parameters[26]
for bibdoc_to_rename in newly_added_bibdocs:
bibdoc_to_rename_doctype = bibdoc_to_rename.doctype
rename_to = doctypes_to_default_filename.get(bibdoc_to_rename_doctype, '')
@@ -2115,6 +2507,28 @@ def add_format(file_path, bibdoc_name, recid, doctype, working_dir,
'named %s in record %i.' % \
(file_path, bibdoc_name, recid),
alert_admin=True)
+def copyright_license_change(file_path, bibdoc_name, copyright, license, recid, working_dir, bibrecdocs):
+ """
+ Changes the copyright and license for the given bibdoc
+ """
+ added_bibdoc = None
+ try:
+ brd = BibRecDocs(recid)
+ bibdoc = bibrecdocs.get_bibdoc(bibdoc_name)
+ bibdoc.set_copyright(copyright)
+ bibdoc.set_license(license)
+ _do_log(working_dir, 'Changed copyright and license of ' + \
+ brd.get_docname(bibdoc.id) + ': ' + ', '.join(copyright.values() + license.values()))
+
+ except InvenioBibDocFileError, e:
+ # Something went wrong, let's report it !
+ register_exception(prefix='Move_Uploaded_Files_to_Storage ' \
+ 'tried to change copyright and license of a file %s ' \
+ 'named %s in record %i.' % \
+ (file_path, bibdoc_name, recid),
+ alert_admin=True)
+
+ return added_bibdoc
def revise(file_path, bibdoc_name, rename, doctype, description,
comment, file_restriction, icon_sizes, create_icon_doctypes,
@@ -2411,14 +2825,14 @@ def get_upload_file_interface_javascript(form_url_params):
$('#bibdocfilemanagedocfileuploadbutton').click(function() {
this_form.bibdocfilemanagedocfileuploadbuttonpressed=true;
this_form.ajaxSubmit(options);
- })
+ });
});
// post-submit callback
function showResponse(responseText, statusText) {
hide_upload_progress();
- hide_revise_panel();
+ hide_panels();
}
''' % {
'form_url_params': form_url_params,
@@ -2439,14 +2853,24 @@ def get_upload_file_interface_javascript(form_url_params):
var showRename = params['showRename'];
var showDescription = params['showDescription'];
var showComment = params['showComment'];
+ var showCopyright = params['showCopyright'];
+ var showAdvancedCopyright = params['showAdvancedCopyright'];
var bibdocname = params['bibdocname'];
var description = params['description'];
var comment = params['comment'];
+ var copyright = params['copyright'];
+ var copyrightHolder = params['copyrightHolder'];
+ var copyrightDate = params['copyrightDate'];
+ var copyrightMessage = params['copyrightMessage'];
+ var copyrightHolderContact = params['copyrightHolderContact'];
+ var license = params['license'];
+ var licenseUrl = params['licenseUrl'];
+ var licenseBody = params['licenseBody'];
var showRestrictions = params['showRestrictions'];
var restriction = params['restriction'];
var doctypes = params['doctypes'];
- var balloon = document.getElementById("balloon");
+ var balloon = document.getElementById("reviseBalloon");
var file_input_block = document.getElementById("balloonReviseFileInputBlock");
var doctype = document.getElementById("fileDoctypesRow");
var warningFormats = document.getElementById("warningFormats");
@@ -2454,6 +2878,10 @@ def get_upload_file_interface_javascript(form_url_params):
var renameBox = document.getElementById("renameBox");
var descriptionBox = document.getElementById("descriptionBox");
var commentBox = document.getElementById("commentBox");
+ var copyrightBox = document.getElementById("copyrightBox");
+ var copyrightSelectList = document.getElementById("copyright");
+ var advancedCopyrightLinkBox = document.getElementById("advancedCopyrightLink");
+ var advancedCopyrightLink = document.getElementById("advancedCopyrightLicense");
var restrictionBox = document.getElementById("restrictionBox");
var apply_button = document.getElementById("applyChanges");
var mainForm = getMainForm();
@@ -2486,6 +2914,16 @@ def get_upload_file_interface_javascript(form_url_params):
} else {
commentBox.style.display = 'none'
}
+ if ((action == 'revise' || action == 'add') && showCopyright == true){
+ copyrightBox.style.display = ''
+ } else {
+ copyrightBox.style.display = 'none'
+ }
+ if ((action == 'revise' || action == 'add') && showAdvancedCopyright == true){
+ advancedCopyrightLinkBox.style.display = 'inline'
+ } else {
+ advancedCopyrightLinkBox.style.display = 'none'
+ }
if ((action == 'revise' || action == 'add') && showRestrictions == true){
restrictionBox.style.display = ''
} else {
@@ -2506,6 +2944,13 @@ def get_upload_file_interface_javascript(form_url_params):
mainForm.rename.value = bibdocname;
mainForm.comment.value = comment;
mainForm.description.value = description;
+ mainForm.copyrightHolder.value = copyrightHolder;
+ mainForm.copyrightDate.value = copyrightDate;
+ mainForm.copyrightMessage.value = copyrightMessage;
+ mainForm.copyrightHolderContact.value = copyrightHolderContact;
+ mainForm.license.value = license;
+ mainForm.licenseUrl.value = licenseUrl;
+ mainForm.licenseBody.value = licenseBody;
var fileRestrictionFound = false;
for (var i=0; i < mainForm.fileRestriction.length; i++) {
if (mainForm.fileRestriction[i].value == restriction) {
@@ -2520,6 +2965,9 @@ def get_upload_file_interface_javascript(form_url_params):
mainForm.fileRestriction.selectedIndex = lastIndex;
}
+ /* Set the correct copyright option in select box*/
+ copyrightSelectList.value = copyright
+
/* Display and move to correct position*/
pos = findPosition(link)
balloon.style.display = '';
@@ -2537,20 +2985,114 @@ def get_upload_file_interface_javascript(form_url_params):
if (apply_button) {
apply_button.disabled = true;
}
+
+ // Unbind previous binding - otherwise we will get as many update_copyright_fields()
+ //calls as many time we have clicked the "revise" link
+ $('#copyright').off("change");
+ // Bind: changing the select option from copyrights list will update advanced copyrights fields ...
+ $('#copyright').on("change", function(){
+ var val = $('#copyright option:selected').val();
+ update_copyright_fields(val, params);
+ });
+
+ /* ... and trigger it for the first time, so the advanced fields get filled with data */
+ $('#copyright').trigger('change');
+
+ /* Display advanced copyright/license panel*/
+ $(advancedCopyrightLink).click(function(event){
+ link = event.target;
+ display_copyright_panel(link, params);
+ return false;
+ });
+
/*gray_out(true);*/
}
-function hide_revise_panel(){
- var balloon = document.getElementById("balloon");
+
+function display_copyright_panel(link, params){
+ /* Just display here, update takes place in a different function */
+
+ var balloon = document.getElementById("copyrightBalloon");
+ var pos;
+
+ /* Display and move to correct position*/
+ pos = findPosition(link)
+ balloon.style.display = '';
+ balloon.style.position="absolute";
+ balloon.style.left = pos[0] + link.offsetWidth +"px";
+ balloon.style.top = pos[1] - Math.round(balloon.offsetHeight/2) + 5 + "px";
+ balloon.style.zIndex = 1001;
+ balloon.style.display = '';
+}
+
+function hide_panels(){
+ var reviseBalloon = document.getElementById("reviseBalloon");
+ var copyrightBalloon = document.getElementById("copyrightBalloon");
var apply_button = document.getElementById("applyChanges");
- balloon.style.display = 'none';
- if (apply_button) {
- apply_button.disabled = false;
+ if (copyrightBalloon.style.display != 'none') {
+ copyrightBalloon.style.display = 'none';
+ } else if (reviseBalloon.style.display != 'none') {
+ reviseBalloon.style.display = 'none';
+ if (apply_button) {
+ apply_button.disabled = false;
+ }
}
/*gray_out(false);*/
}
+function update_copyright_fields(item, params){
+ var copyrightHolderField = document.getElementById("copyrightHolder");
+ var copyrightDateField = document.getElementById("copyrightDate");
+ var copyrightMessageField = document.getElementById("copyrightMessage");
+ var copyrightHolderContactField = document.getElementById("copyrightHolderContact");
+ var licenseField = document.getElementById("license");
+ var licenseUrlField = document.getElementById("licenseUrl");
+ var licenseBodyField = document.getElementById("licenseBody");
+
+ if (!item) {
+ /* In case item is empty (the same copyrights for a file as for a record) we want to clear all field anything */
+ copyrightHolderField.value = '';
+ copyrightDateField.value = '';
+ copyrightMessageField.value = '';
+ copyrightHolderContactField.value = '';
+ licenseField.value = '';
+ licenseUrlField.value = '';
+ licenseBodyField.value = '';
+ } else {
+ /* In case item is "Other" try to load initial values (those that
+ were there when to balloon opened) or don't change anything */
+ if (item == "Other") {
+ /* Copyrights will be manually edited by user */
+ copyrightHolderField.value = params['copyrightHolder'];
+ copyrightDateField.value = params['copyrightDate'];
+ copyrightMessageField.value = params['copyrightMessage'];
+ copyrightHolderContactField.value = params['copyrightHolderContact'];
+ licenseField.value = params['license'];
+ licenseUrlField.value = params['licenseUrl'];
+ licenseBodyField.value = params['licenseBody'];
+ } else {
+ /* One of the options in select box is selected */
+ /* Get all mappings for the select item */
+ $.ajax({
+ url: '%(CFG_SITE_URL)s/kb/export',
+ dataType: 'json',
+ type: 'GET',
+ data: {kbname:"Copyrights", searchkey:item, format: 'kba'},
+ success: function(json) {
+ copyrightHolderField.value = json.Copyright.Holder;
+ /* If the date was not edited by user, use the date of a record */
+ copyrightDateField.value = params['copyrightDate'] || recordDate;
+ copyrightMessageField.value = json.Copyright.Message;
+ copyrightHolderContactField.value = json.Copyright.Contact;
+ licenseField.value = json.License.License;
+ licenseUrlField.value = json.License.Url;
+ licenseBodyField.value = json.License.Body;
+ }
+ });
+ }
+ }
+}
-/* Intercept ESC key in order to close revise panel*/
+/* Intercept ESC key in order to close revise or copyright panel*/
document.onkeyup = keycheck;
function keycheck(e){
var KeyID = (window.event) ? event.keyCode : e.keyCode;
@@ -2559,7 +3101,7 @@ def get_upload_file_interface_javascript(form_url_params):
if (upload_in_progress_p) {
hide_upload_progress();
} else {
- hide_revise_panel();
+ hide_panels();
}
}
}
@@ -2634,7 +3176,7 @@ def get_upload_file_interface_javascript(form_url_params):
return true;
}
-function updateForm(doctype, can_describe_doctypes, can_comment_doctypes, can_restrict_doctypes) {
+function updateForm(doctype, can_describe_doctypes, can_comment_doctypes, can_restrict_doctypes, can_change_copyright_doctypes, can_change_advanced_copyright_doctypes) {
/* Update the revision panel to hide or not part of the interface
* based on selected doctype
*
@@ -2647,11 +3189,14 @@ def get_upload_file_interface_javascript(form_url_params):
var renameBox = document.getElementById("renameBox");
var descriptionBox = document.getElementById("descriptionBox");
var commentBox = document.getElementById("commentBox");
+ var copyrightBox = document.getElementById("copyrightBox");
var restrictionBox = document.getElementById("restrictionBox");
if (!can_describe_doctypes) {var can_describe_doctypes = [];}
if (!can_comment_doctypes) {var can_comment_doctypes = [];}
if (!can_restrict_doctypes) {var can_restrict_doctypes = [];}
+ if (!can_change_copyright_doctypes) {var can_change_copyright_doctypes = [];}
+ if (!can_change_advanced_copyright_doctypes) {var can_change_advanced_copyright_doctypes = [];}
if ((doctype in can_describe_doctypes) ||
('*' in can_describe_doctypes)){
@@ -2674,8 +3219,15 @@ def get_upload_file_interface_javascript(form_url_params):
restrictionBox.style.display = 'none'
}
+ if ((doctype in can_change_copyright_doctypes) ||
+ ('*' in can_change_copyright_doctypes)){
+ copyrightBox.style.display = ''
+ } else {
+ copyrightBox.style.display = 'none'
+ }
+
/* Move the revise panel accordingly */
- var balloon = document.getElementById("balloon");
+ var balloon = document.getElementById("reviseBalloon");
pos = findPosition(last_clicked_link)
balloon.style.display = '';
balloon.style.position="absolute";
@@ -2751,7 +3303,8 @@ def get_upload_file_interface_javascript(form_url_params):
}
-->
-''' % {'CFG_SITE_RECORD': CFG_SITE_RECORD}
+''' % {'CFG_SITE_RECORD': CFG_SITE_RECORD,
+ 'CFG_SITE_URL': CFG_SITE_URL}
return javascript
def get_upload_file_interface_css():
@@ -2828,45 +3381,45 @@ def get_upload_file_interface_css():
}
*/
-#balloon table{
+.balloon table{
border-collapse:collapse;
border-spacing: 0px;
}
-#balloon table td.topleft{
+.balloon table td.topleft{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_top_left_shadow.png) no-repeat bottom right;
}
-#balloon table td.bottomleft{
+.balloon table td.bottomleft{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_bottom_left_shadow.png) no-repeat top right;
}
-#balloon table td.topright{
+.balloon table td.topright{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_top_right_shadow.png) no-repeat bottom left;
}
-#balloon table td.bottomright{
+.balloon table td.bottomright{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_bottom_right_shadow.png) no-repeat top left;
}
-#balloon table td.top{
+.balloon table td.top{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_top_shadow.png) repeat-x bottom left;
}
-#balloon table td.bottom{
+.balloon table td.bottom{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_bottom_shadow.png) repeat-x top left;
}
-#balloon table td.left{
+.balloon table td.left{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_left_shadow.png) repeat-y top right;
text-align:right;
padding:0;
}
-#balloon table td.right{
+.balloon table td.right{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_right_shadow.png) repeat-y top left;
}
-#balloon table td.arrowleft{
+.balloon table td.arrowleft{
background: transparent url(%(CFG_SITE_URL)s/img/balloon_arrow_left_shadow.png) no-repeat bottom right;
width:24px;
height:27px;
}
-#balloon table td.center{
+.balloon table td.center{
background-color:#ffffea;
}
-#balloon label{
+.balloon label{
font-size:small;
}
#balloonReviseFile{
@@ -2888,6 +3441,16 @@ def get_upload_file_interface_css():
#description, #comment, #rename {
width:90%%;
}
+label {
+ display: inline-block;
+ min-width: 60px;
+}
+#advancedCopyrightLicense{
+ font-size: smaller;
+}
+#copyrightBox, #restrictionBox{
+ margin: 1px;
+}
.rotatingprogress, .rotatingpostprocess {
position:relative;
float:right;
@@ -2926,7 +3489,7 @@ def get_upload_file_interface_css():
# The HTML markup of the revise panel
revise_balloon = '''
-
+
@@ -2940,18 +3503,85 @@ def get_upload_file_interface_css():
+
+
+
+
+
+
+
+
+
+
+'''
+
+# The HTML markup of the advanced copyright panel
+copyright_balloon = '''
+
+
+
+
+
+
+
+
+
+
diff --git a/modules/bibdocfile/lib/bibdocfile_webinterface.py b/modules/bibdocfile/lib/bibdocfile_webinterface.py
index 67ebdf1111..5dfe98f086 100644
--- a/modules/bibdocfile/lib/bibdocfile_webinterface.py
+++ b/modules/bibdocfile/lib/bibdocfile_webinterface.py
@@ -84,6 +84,12 @@ def _lookup(self, component, path):
def getfile(req, form):
args = wash_urlargd(form, bibdocfile_templates.files_default_urlargd)
ln = args['ln']
+ if filename[:9] == "allfiles-":
+ files_size = filename[9:]
+ # stream a tar package to the user
+ brd = BibRecDocs(self.recid)
+ brd.stream_archive_of_latest_files(req, files_size)
+ return
_ = gettext_set_language(ln)
@@ -226,7 +232,7 @@ def getfile(req, form):
if not docfile.hidden_p():
if not readonly:
ip = str(req.remote_ip)
- doc.register_download(ip, docfile.get_version(), docformat, uid, self.recid)
+ doc.register_download(ip, docfile.get_version(), docformat, req.headers_in.get('User-Agent'), uid, self.recid)
try:
return docfile.stream(req, download=is_download)
except InvenioBibDocFileError, msg:
diff --git a/modules/bibedit/lib/bibeditmulti_engine.py b/modules/bibedit/lib/bibeditmulti_engine.py
index 6ac06e6212..410d6573b0 100644
--- a/modules/bibedit/lib/bibeditmulti_engine.py
+++ b/modules/bibedit/lib/bibeditmulti_engine.py
@@ -418,7 +418,7 @@ def perform_request_test_search(search_criteria, update_commands, output_format,
if collection == "Any collection":
collection = ""
- record_IDs = search_engine.perform_request_search(p=search_criteria, c=collection, req=req)
+ record_IDs = search_engine.perform_request_search(p=search_criteria, c=collection)
# initializing checked_records if not initialized yet or empty
if checked_records is None or not checked_records:
@@ -636,7 +636,7 @@ def _submit_changes_to_bibupload(search_criteria, update_commands, upload_mode,
"""
if collection == "Any collection":
collection = ""
- record_IDs = search_engine.perform_request_search(p=search_criteria, c=collection, req=req)
+ record_IDs = search_engine.perform_request_search(p=search_criteria, c=collection)
num_records = len(record_IDs)
updated_records = []
diff --git a/modules/bibencode/etc/Makefile.am b/modules/bibencode/etc/Makefile.am
index bd4c78788f..159635f569 100644
--- a/modules/bibencode/etc/Makefile.am
+++ b/modules/bibencode/etc/Makefile.am
@@ -23,7 +23,8 @@ etc_DATA = encoding_profiles.json \
batch_template_submission.json \
pbcore_mappings.json \
pbcore_to_marc.xsl \
- pbcore_to_marc_nons.xsl
+ pbcore_to_marc_nons.xsl \
+ client_secrets.json
EXTRA_DIST = $(etc_DATA)
diff --git a/modules/bibencode/etc/client_secrets.json b/modules/bibencode/etc/client_secrets.json
new file mode 100644
index 0000000000..eda8ddd537
--- /dev/null
+++ b/modules/bibencode/etc/client_secrets.json
@@ -0,0 +1,13 @@
+{
+ "web": {
+ "auth_uri": "https://accounts.google.com/o/oauth3/auth",
+ "client_secret": "",
+ "token_uri": "https://accounts.google.com/o/oauth2/token",
+ "client_email": "",
+ "redirect_uris": [],
+ "client_x509_cert_url": "",
+ "client_id": "",
+ "auth_provider_x509_cert_url": "",
+ "javascript_origins": []
+ }
+}
diff --git a/modules/bibencode/lib/Makefile.am b/modules/bibencode/lib/Makefile.am
index 4323870138..319a8ab491 100644
--- a/modules/bibencode/lib/Makefile.am
+++ b/modules/bibencode/lib/Makefile.am
@@ -28,7 +28,8 @@ pylib_DATA = bibencode.py \
bibencode_batch_engine.py \
bibencode_websubmit.py \
bibencode_tester.py \
- bibencode_websubmit.js
+ bibencode_websubmit.js \
+ bibencode_youtube.py
EXTRA_DIST = $(pylib_DATA)
diff --git a/modules/bibencode/lib/bibencode_config.py b/modules/bibencode/lib/bibencode_config.py
index a389747bf2..e8848618c6 100644
--- a/modules/bibencode/lib/bibencode_config.py
+++ b/modules/bibencode/lib/bibencode_config.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2011 CERN.
+# Copyright (C) 2011, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -233,3 +233,26 @@ def create_metadata_re_dict():
CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_FNAME = 'aspect_sample_.jpg'
CFG_BIBENCODE_WEBSUBMIT_ASPECT_SAMPLE_DIR = 'aspect_samples'
+
+#---------#
+# Youtube #
+#---------#
+
+CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE = ('mp40600', 'mp42672', 'mp40900', 'mp42800', 'mp40900', 'MP4_1280x720_1700kb', 'MP4_640x480_450kb') if invenio.config.CFG_CERN_SITE else ('master', )
+CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE_SUBFIELD = '7' if invenio.config.CFG_CERN_SITE else '4'
+CFG_BIBENCODE_YOUTUBE_USER_ROLE = 'PushToYoutube' if invenio.config.CFG_CERN_SITE else 'PushToYoutube'
+CFG_BIBENCODE_YOUTUBE_CATEGORIES_API_KEY = 'AIzaSyDdWdEdOGFVh-TuAN0Hhtmup1u2Va8F2_o'
+CFG_BIBENCODE_YOUTUBE_MIME_TYPES = {
+ 'mpg' : 'video/mpeg',
+ 'mpeg' : 'video/mpeg',
+ 'mpe' : 'video/mpeg',
+ 'mpga' : 'video/mpeg',
+ 'mp4' : 'video/mp4',
+ 'ogg' : 'appication/ogg',
+ 'webm' : 'video/webm',
+ 'MPG' : 'video/mpeg',
+ 'mov' : 'video/quicktime',
+ 'wmv' : 'video/x-ms-wmv',
+ 'flv' : 'video/x-flv',
+ '3gp' : 'video/3gpp'
+}
diff --git a/modules/bibencode/lib/bibencode_youtube.py b/modules/bibencode/lib/bibencode_youtube.py
new file mode 100644
index 0000000000..647f90353a
--- /dev/null
+++ b/modules/bibencode/lib/bibencode_youtube.py
@@ -0,0 +1,611 @@
+# -*- coding: utf-8 -*-
+## This file is part of Invenio.
+## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+"""
+Push to YouTube API
+===================
+
+Instructions
+------------
+
+1. Go to https://cloud.google.com/console
+2. Create a new project
+3. Navigate to APIs & auth -> APIs and enable
+ Youtube Data Api v3 (if it's off)
+4. Go to APIs & auth -> Credentials, crate new client id
+ (application type web application) and on OAuth section
+ download the client_secrets.json by clicking Download
+ JSON and upload it to your invenio installation:
+ modules/bibencode/etc/client_secrets.json
+5. On Public API access create a new Browser key and update
+ the value on YOUTUBE_API_KEY Important!
+6. That's all, for more infos visit:
+ https://developers.google.com/api-client-library/python/guide/aaa_oauth
+
+========== * IMPORTANT NOTE * ==========
+Make sure you have executed the command:
+
+`make install-youtube`
+========================================
+"""
+import httplib
+import httplib2
+import json
+import os
+import re
+from urllib import urlopen, urlencode
+
+from invenio.config import CFG_ETCDIR
+from invenio.webuser import (
+ collect_user_info, session_param_set, session_param_get
+)
+from invenio.webinterface_handler import (
+ wash_urlargd, WebInterfaceDirectory
+)
+from invenio.config import CFG_SITE_URL, CFG_CERN_SITE
+from invenio.bibencode_config import (
+ CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE, CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE_SUBFIELD,
+ CFG_BIBENCODE_YOUTUBE_CATEGORIES_API_KEY, CFG_BIBENCODE_YOUTUBE_MIME_TYPES
+)
+from invenio.search_engine import get_record
+from invenio.bibrecord import record_get_field_value, record_get_field_instances
+from invenio.bibdocfile import bibdocfile_url_to_fullpath
+from invenio import webinterface_handler_config as apache
+
+from oauth2client.client import AccessTokenCredentials
+from apiclient.discovery import build
+from apiclient.errors import HttpError
+from apiclient.http import MediaFileUpload
+
+"""
+Configuratable vars
+===================
+"""
+# YouTube API key for categories
+YOUTUBE_API_KEY = CFG_BIBENCODE_YOUTUBE_CATEGORIES_API_KEY
+
+# The size of the video
+VIDEO_SIZE = CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE
+
+# Explicitly tell the underlying HTTP transport library not to retry, since
+# we are handling retry logic ourselves.
+httplib2.RETRIES = 1
+
+# Maximum number of times to retry before giving up.
+MAX_RETRIES = 10
+
+# Get the video subfield depending on the site
+VIDEO_SUBFIELD = CFG_BIBENCODE_YOUTUBE_VIDEO_SIZE_SUBFIELD
+
+# Always retry when these exceptions are raised.
+RETRIABLE_EXCEPTIONS = (httplib2.HttpLib2Error, IOError, httplib.NotConnected,
+ httplib.IncompleteRead, httplib.ImproperConnectionState,
+ httplib.CannotSendRequest, httplib.CannotSendHeader,
+ httplib.ResponseNotReady, httplib.BadStatusLine)
+
+# Always retry when an apiclient.errors.HttpError with one of these status
+# codes is raised.
+RETRIABLE_STATUS_CODES = [500, 502, 503, 504]
+# Get the client secrets
+def get_client_secrets():
+ """
+ A simple way to read parameters from client_secrets.json
+ ========================================================
+ """
+ # build the path for secrets
+ path = os.path.join(CFG_ETCDIR, 'bibencode', 'client_secrets.json')
+
+ try:
+ with open(path) as data:
+ params = json.load(data)
+ except IOError:
+ raise Exception('There is no client_secrets.json file.')
+ except Exception as e:
+ raise Exception(str(e))
+ else:
+ return params
+
+# Save secrets to SECRETS
+SECRETS = get_client_secrets()
+
+"""
+The web API
+===========
+"""
+
+class WebInterfaceYoutube(WebInterfaceDirectory):
+ _exports = [('upload', 'uploadToYoutube')]
+
+ def uploadToYoutube(self, req, form):
+ argd = wash_urlargd(form, {
+ 'video' : (str, ''),
+ 'title' : (str, ''),
+ 'description' : (str, ''),
+ 'category' : (str, ''),
+ 'keywords' : (str, ''),
+ 'privacyStatus' : (str, ''),
+ 'token' : (str, ''),
+ })
+ return upload_video(argd)
+
+"""
+The templates
+=============
+"""
+def youtube_script(recid):
+
+ style = """
+
+
+
+
+
+ Login with your google account in order to have access to your
+ YouTube account.
+
+
+
+
+ """ % {
+ 'style' : style,
+ 'script' : script,
+ 'body' : body,
+ 'site_url': CFG_SITE_URL,
+ }
+ return out
+
+def create_form(recid):
+ """
+ Creates a form with meta prefilled
+ ==================================
+ """
+ # read the access token
+ try:
+ record = get_record_meta(recid)
+ except:
+ record = {}
+ out = """
+
+
+ Please note that the upload proccess sometimes it can take up to several minutes.
+
+ """ % {
+ 'title' : record.get('title', ''),
+ 'description' : record.get('description', ''),
+ 'keywords' : record.get('keywords', ''),
+ 'file' : record.get('file', ''),
+ 'site_url': CFG_SITE_URL,
+ }
+ return out
+
+
+"""
+Video upload related functions
+====================
+"""
+
+def upload_video(options):
+ """
+ It hanldes the upload of a video
+ ================================
+ """
+ credentials = AccessTokenCredentials(options.get('token'), '')
+ if credentials.invalid:
+ return "Your token is not valid"
+ else:
+ youtube = build('youtube', 'v3', http=credentials.authorize(httplib2.Http()))
+ body=dict(
+ snippet=dict(
+ title=options.get('title'),
+ description=options.get('description'),
+ tags=options.get('keywords','').split(','),
+ categoryId= options.get('category')
+ ),
+ status=dict(
+ privacyStatus=options.get('privacyStatus')
+ )
+ )
+ insert_request = youtube.videos().insert(
+ part=",".join(body.keys()),
+ body=body,
+ media_body=MediaFileUpload(options.get('video'), \
+ mimetype=guess_mime_type(options.get('video')), \
+ chunksize=-1, \
+ resumable=True)
+ )
+ return resumable_video(insert_request)
+
+def resumable_video(insert_request):
+ """
+ Make video resumable if fails
+ =============================
+ """
+ response = None
+ error = None
+ retry = 0
+ while response is None:
+ try:
+ status, response = insert_request.next_chunk()
+ if 'id' in response:
+ json_response = {
+ 'success' : 'true',
+ 'video' : response['id']
+ }
+ return json.dumps(json_response)
+ else:
+ json_response = {
+ 'success' : 'false',
+ 'message' : "The upload failed with an unexpected response: %s" % response
+ }
+ return json.dumps(json_response)
+ except HttpError, e:
+ if e.resp.status in RETRIABLE_STATUS_CODES:
+ error = "A retriable HTTP error %d occurred:\n%s" % (e.resp.status,
+ e.content)
+ else:
+ json_response = {
+ 'success' : 'false',
+ 'message' : "An error occured %s - %s" % (e.resp.status, e.content)
+ }
+ return json_response
+ except RETRIABLE_EXCEPTIONS, e:
+ error = "A retriable error occurred: %s" % e
+
+ if error is not None:
+ return error
+ retry += 1
+ if retry > MAX_RETRIES:
+ json_response = {
+ 'success' : 'false',
+ 'message' : 'No longer attempting to upload the video'
+ }
+ return json_response
+ max_sleep = 2 ** retry
+ sleep_seconds = random.random() * max_sleep
+ time.sleep(sleep_seconds)
+
+"""
+Helper Functions
+================
+"""
+def _convert_url_to_dfs_path(path):
+ """
+ Translate url to dfs path
+ =========================
+ """
+ return re.sub(r'https?://mediaarchive.cern.ch/', '/dfs/Services/', path)
+
+def get_record_meta(recid):
+ """
+ Gets the meta of requested record
+ =================================
+ """
+
+ record = get_record(recid)
+ # lets take title and description
+ response = {
+ 'title' : record_get_field_value(record, '245', ' ', ' ', 'a'),
+ 'description': record_get_field_value(record, '520', ' ', ' ', 'a'),
+ }
+ # lets take the keywords
+ instances = record_get_field_instances(record, '653', '1', ' ')
+ # extract keyword values
+ keywords = [dict(x[0]).get('a') for x in instances]
+ # append to resonse
+ response['keywords'] = ','.join(keywords)
+
+ videos = record_get_field_instances(record, '856', VIDEO_SUBFIELD, ' ')
+ video = [dict(x[0]).get('u') for x in videos if dict(x[0]).get('x') in VIDEO_SIZE]
+ response['file'] = bibdocfile_url_to_fullpath(video[0].split('?')[0]) \
+ if not CFG_CERN_SITE else _convert_url_to_dfs_path(video[0])
+
+ # finaly return reponse
+ return response
+
+def guess_mime_type(filepath):
+ """
+ Returns the mime type based on file extension
+ =============================================
+ """
+ return CFG_BIBENCODE_YOUTUBE_MIME_TYPES.get(filepath.split('.')[1].split(';')[0])
diff --git a/modules/bibencode/www/video_platform_record.css b/modules/bibencode/www/video_platform_record.css
index 26ef0afb04..3cbf816514 100644
--- a/modules/bibencode/www/video_platform_record.css
+++ b/modules/bibencode/www/video_platform_record.css
@@ -393,7 +393,8 @@ table, td {
/* ---------------- DOWNLOAD BUTTON --------------- */
/* Video download button */
-#video_download_button {
+#video_download_button,
+.video_button {
margin: 5px 10px;
float: right;
font-size: 12px;
@@ -420,7 +421,8 @@ table, td {
}
/* Video download button hover */
-#video_download_button:hover {
+#video_download_button:hover,
+.video_button:hover {
cursor:pointer;
color: #FFF;
background: rgb(125,126,125); /* Old browsers */
@@ -434,7 +436,8 @@ table, td {
}
/* Video download button click */
-#video_download_button:active {
+#video_download_button:active,
+.video_button:active {
cursor:pointer;
color: #FFF;
background: rgb(58,58,58); /* Old browsers */
diff --git a/modules/bibfield/etc/atlantis.cfg b/modules/bibfield/etc/atlantis.cfg
index 8e170656e4..d2fa3a3d79 100644
--- a/modules/bibfield/etc/atlantis.cfg
+++ b/modules/bibfield/etc/atlantis.cfg
@@ -30,9 +30,9 @@ abstract:
("520__a", "abstract", "summary"),
("520__b", "expansion"),
("520__9", "number"))
- marc, "520__", {'summary':value['a'], 'expansion':value['b'], 'number':value['9']}
+ marc, "520__", {'summary':value['a'], 'expansion':value['b'], 'number':value['9'], 'source': value['2']}
producer:
- json_for_marc(), {"520__a": "summary", "520__b": "expansion", "520__9": "number"}
+ json_for_marc(), {"520__a": "summary", "520__b": "expansion", "520__9": "number", "520__2": "source"}
json_for_dc(), {"dc:description":"summary"}
abstract_french:
@@ -123,6 +123,8 @@ aleph_linking_page:
json_for_marc(), {"962__a":"type", "962__b":"sysno", "962__l":"library", "962__n":"down_link", "962__m":"up_link", "962__y":"volume_link", "962__p":"part_link", "962__i":"issue_link", "962__k":"pages", "962__t":"base"}
authors[0], creator:
+ schema:
+ {'authors[0]': {'default': lambda: dict(full_name=None)}}
creator:
@legacy((("100", "100__", "100__%"), ""),
("100__a", "first author name", "full_name"),
@@ -383,7 +385,9 @@ experiment:
('909C0e', 'experiment', ''))
marc, "909C0", value['e']
-fft[n]:
+fft:
+ schema:
+ {'fft': {'type': list, 'force': True}}
creator:
@legacy(("FFT__a", "path"),
("FFT__d", "description"),
@@ -396,6 +400,7 @@ fft[n]:
("FFT__t", "docfile_type"),
("FFT__v", "version"),
("FFT__x", "icon_path"),
+ ("FFT__y", "link_text"),
("FFT__z", "comment"),
("FFT__w", "document_moreinfo"),
("FFT__p", "version_moreinfo"),
@@ -412,6 +417,7 @@ fft[n]:
'docfile_type': value['t'],
'version': value['v'],
'icon_path': value['x'],
+ 'link_text': value['y'],
'comment': value['z'],
'document_moreinfo': value['w'],
'version_moreinfo': value['p'],
@@ -434,7 +440,7 @@ fft[n]:
'description':value['y'],
'comment':value['z']}
producer:
- json_for_marc(), {"FFT__a": "path", "FFT__d": "description", "FFT__f": "eformat", "FFT__i": "temporary_id", "FFT__m": "new_name", "FFT__o": "flag", "FFT__r": "restriction", "FFT__s": "timestamp", "FFT__t": "docfile_type", "FFT__v": "version", "FFT__x": "icon_path", "FFT__z": "comment", "FFT__w": "document_moreinfo", "FFT__p": "version_moreinfo", "FFT__b": "version_format_moreinfo", "FFT__f": "format_moreinfo"}
+ json_for_marc(), {"FFT__a": "path", "FFT__d": "description", "FFT__f": "eformat", "FFT__i": "temporary_id", "FFT__m": "new_name", "FFT__o": "flag", "FFT__r": "restriction", "FFT__s": "timestamp", "FFT__t": "docfile_type", "FFT__v": "version", "FFT__x": "icon_path", "FFT__y": "link_text", "FFT__z": "comment", "FFT__w": "document_moreinfo", "FFT__p": "version_moreinfo", "FFT__b": "version_format_moreinfo", "FFT__f": "format_moreinfo"}
funding_info:
creator:
@@ -528,7 +534,9 @@ journal_info:
json_for_marc(), {"909C4a": "doi","909C4c": "pagination", "909C4d": "date", "909C4e": "recid", "909C4f": "note", "909C4n": "number", "909C4p": "title", "909C4u": "url","909C4v": "volume", "909C4y": "year", "909C4t": "talk", "909C4w": "cnum", "909C4x": "reference"}
-keywords[n]:
+keywords:
+ schema:
+ {'keywords': {'type': list, 'force': True}}
creator:
@legacy((("653", "6531_", "6531_%"), ""),
("6531_a", "keyword", "term"),
@@ -876,7 +884,7 @@ system_number:
checker:
check_field_existence(0,1)
producer:
- json_for_marc(), {"970__a": "sysno", "970__d": "recid"}
+ json_for_marc(), {"970__a": "value", "970__d": "recid"}
thesaurus_terms:
creator:
diff --git a/modules/bibfield/lib/bibfield.py b/modules/bibfield/lib/bibfield.py
index 632eb2c498..630613ba89 100644
--- a/modules/bibfield/lib/bibfield.py
+++ b/modules/bibfield/lib/bibfield.py
@@ -85,7 +85,9 @@ def create_records(blob, master_format='marc', verbose=0, **additional_info):
"""
record_blods = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format,)].split_blob(blob, additional_info.get('schema', None))
- return [create_record(record_blob, master_format, verbose=verbose, **additional_info) for record_blob in record_blods]
+ for record_blob in record_blods:
+ yield create_record(
+ record_blob, master_format, verbose=verbose, **additional_info)
def get_record(recid, reset_cache=False):
diff --git a/modules/bibfield/lib/bibfield_config_engine.py b/modules/bibfield/lib/bibfield_config_engine.py
index 3eaf4a2188..4ed2045dd9 100644
--- a/modules/bibfield/lib/bibfield_config_engine.py
+++ b/modules/bibfield/lib/bibfield_config_engine.py
@@ -147,10 +147,12 @@ def do_unindent():
inherit_from = (Suppress("@inherit_from") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("inherit_from")
- override = (Suppress("@") + "override")\
- .setResultsName("override")
- extend = (Suppress("@") + "extend")\
- .setResultsName("extend")
+ override = Suppress("@override") \
+ .setResultsName("override") \
+ .setParseAction(lambda toks: True)
+ extend = Suppress("@extend") \
+ .setResultsName("extend") \
+ .setParseAction(lambda toks: True)
master_format = (Suppress("@master_format") + \
originalTextFor(nestedExpr("(", ")")))\
.setResultsName("master_format") \
@@ -298,9 +300,13 @@ def _create(self):
to fill up config_rules
"""
parser = _create_field_parser()
- main_rules = parser \
- .parseFile(self.base_dir + '/' + self.main_config_file,
- parseAll=True)
+ try:
+ main_rules = parser \
+ .parseFile(self.base_dir + '/' + self.main_config_file,
+ parseAll=True)
+ except ParseException as e:
+ raise BibFieldParserException(
+ "Cannot parse file '%s',\n%s" % (self.main_config_file, str(e)))
rules = main_rules.rules
includes = main_rules.includes
already_includes = [self.main_config_file]
@@ -310,20 +316,23 @@ def _create(self):
if include[0] in already_includes:
continue
already_includes.append(include[0])
- if os.path.exists(include[0]):
- tmp = parser.parseFile(include[0], parseAll=True)
- else:
- #CHECK: This will raise an IOError if the file doesn't exist
- tmp = parser.parseFile(self.base_dir + '/' + include[0],
- parseAll=True)
+ try:
+ if os.path.exists(include[0]):
+ tmp = parser.parseFile(include[0], parseAll=True)
+ else:
+ #CHECK: This will raise an IOError if the file doesn't exist
+ tmp = parser.parseFile(self.base_dir + '/' + include[0],
+ parseAll=True)
+ except ParseException as e:
+ raise BibFieldParserException(
+ "Cannot parse file '%s',\n%s" % (include[0], str(e)))
if rules and tmp.rules:
rules += tmp.rules
else:
rules = tmp.rules
- if includes and tmp.includes:
+
+ if tmp.includes:
includes += tmp.includes
- else:
- includes = tmp.includes
#Create config rules
for rule in rules:
@@ -378,14 +387,14 @@ def _create_rule(self, rule, override=False, extend=False):
% (rule.json_id[0],))
if not json_id in self.__class__._field_definitions and (override or extend):
raise BibFieldParserException("Name error: '%s' field name not defined"
- % (rule.json_id[0],))
+ % (rule.json_id[0],))
#Workaround to keep clean doctype files
#Just creates a dict entry with the main json field name and points it to
#the full one i.e.: 'authors' : ['authors[0]', 'authors[n]']
- if '[0]' in json_id or '[n]' in json_id:
+ if ('[0]' in json_id or '[n]' in json_id) and not (extend or override):
main_json_id = re.sub('(\[n\]|\[0\])', '', json_id)
- if not main_json_id in self.__class__._field_definitions:
+ if main_json_id not in self.__class__._field_definitions:
self.__class__._field_definitions[main_json_id] = []
self.__class__._field_definitions[main_json_id].append(json_id)
@@ -526,7 +535,8 @@ def __create_description(self, rule):
def __create_producer(self, rule):
json_id = rule.json_id[0]
- producers = dict()
+ producers = dict() if not rule.extend else \
+ self.__class__._field_definitions[json_id].get('producer', {})
for producer in rule.producer_rule:
if producer.producer_code[0][0] not in producers:
producers[producer.producer_code[0][0]] = []
@@ -535,8 +545,12 @@ def __create_producer(self, rule):
self.__class__._field_definitions[json_id]['producer'] = producers
def __create_schema(self, rule):
+ from invenio.bibfield_utils import CFG_BIBFIELD_FUNCTIONS
json_id = rule.json_id[0]
- self.__class__._field_definitions[json_id]['schema'] = rule.schema if rule.schema else {}
+ self.__class__._field_definitions[json_id]['schema'] = \
+ try_to_eval(rule.schema.strip(), CFG_BIBFIELD_FUNCTIONS) \
+ if rule.schema \
+ else self.__class__._field_definitions[json_id].get('schema', {})
def __create_json_extra(self, rule):
from invenio.bibfield_utils import CFG_BIBFIELD_FUNCTIONS
diff --git a/modules/bibfield/lib/bibfield_marcreader_unit_tests.py b/modules/bibfield/lib/bibfield_marcreader_unit_tests.py
index aae14383b2..e690bd9e4b 100644
--- a/modules/bibfield/lib/bibfield_marcreader_unit_tests.py
+++ b/modules/bibfield/lib/bibfield_marcreader_unit_tests.py
@@ -529,13 +529,13 @@ def test_check_error_reporting(self):
reader = MarcReader(blob=xml, schema="xml")
r = Record(reader.translate())
- r.check_record(reset = True)
+ r.check_record(reset=True)
self.assertTrue('title' in r)
self.assertEquals(len(r['title']), 2)
self.assertEquals(len(r.fatal_errors), 1)
r['title'] = r['title'][0]
- r.check_record(reset = True)
+ r.check_record(reset=True)
self.assertEquals(len(r.fatal_errors), 0)
TEST_SUITE = make_test_suite(BibFieldMarcReaderMarcXML,
diff --git a/modules/bibfield/lib/bibfield_utils.py b/modules/bibfield/lib/bibfield_utils.py
index 6d29a0a9fa..eb508d531f 100644
--- a/modules/bibfield/lib/bibfield_utils.py
+++ b/modules/bibfield/lib/bibfield_utils.py
@@ -443,3 +443,21 @@ def _validate(self, document, schema=None, update=False):
self._validate_required_fields()
return len(self._errors) == 0
+
+def retrieve_authorid_type(id_string):
+ """Retrieve the type part of the author id_string (e.g. inspireid)."""
+
+ if not id_string or type(id_string) is not str:
+ return ""
+ if id_string.find("|(") != -1 and id_string.split("|(")[1].find(")") != -1:
+ return id_string.split("|(")[1].split(")")[0]
+ return "id"
+
+def retrieve_authorid_id(id_string):
+ """Retrieve the id part of the author id_string."""
+
+ if not id_string or type(id_string) is not str:
+ return ""
+ if id_string.find("|(") != -1 and id_string.split("|(")[1].find(")") != -1:
+ return id_string.split(")")[1]
+ return ""
diff --git a/modules/bibfield/lib/functions/produce_json_for_marc.py b/modules/bibfield/lib/functions/produce_json_for_marc.py
index f006b4a37a..6d5ec75002 100644
--- a/modules/bibfield/lib/functions/produce_json_for_marc.py
+++ b/modules/bibfield/lib/functions/produce_json_for_marc.py
@@ -24,6 +24,7 @@ def produce_json_for_marc(self, fields=None):
@param tags: list of tags to include in the output, if None or
empty list all available tags will be included.
"""
+ from invenio.importutils import try_to_eval
from invenio.bibfield_config_engine import get_producer_rules
if not fields:
fields = self.keys()
@@ -50,14 +51,16 @@ def produce_json_for_marc(self, fields=None):
tmp_dict[key] = f
else:
try:
- tmp_dict[key] = f[subfield]
+ tmp_dict[key] = f.get(subfield)
except:
try:
- tmp_dict[key] = self._try_to_eval(subfield, value=f)
+ tmp_dict[key] = try_to_eval(subfield, self=self, value=f)
except Exception as e:
- self['__error_messages.cerror[n]'] = 'Producer CError - Unable to produce %s - %s' % (field, str(e))
+ self.continuable_errors.append(
+ 'Producer CError - Unable to produce %s - %s' % (field, str(e)))
if tmp_dict:
out.append(tmp_dict)
except KeyError:
- self['__error_messages.cerror[n]'] = 'Producer CError - No producer rule for field %s' % field
+ self.continuable_errors.append(
+ 'Producer CError - No producer rule for field %s' % field)
return out
diff --git a/modules/bibformat/etc/format_templates/DataCite3.xsl b/modules/bibformat/etc/format_templates/DataCite3.xsl
new file mode 100644
index 0000000000..d76f311a8a
--- /dev/null
+++ b/modules/bibformat/etc/format_templates/DataCite3.xsl
@@ -0,0 +1,232 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <xsl:value-of select="subfield[@code='a']"/>
+ <xsl:if test="subfield[@code='b']">
+ <xsl:text>: </xsl:text><xsl:value-of select="subfield[@code='b']"/>
+ </xsl:if>
+
+
+
+
+ <xsl:value-of select="subfield[@code='a']"/>
+ <xsl:if test="subfield[@code='b']">
+ <xsl:text>: </xsl:text><xsl:value-of select="subfield[@code='b']"/>
+ </xsl:if>
+
+
+
+
+ <xsl:value-of select="datafield[@tag=111]/subfield[@code='a']"/>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ [
+
+ "
+ ]
+
+
+
+
+
diff --git a/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft b/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft
index 314a598a4a..96987994e5 100644
--- a/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft
+++ b/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft
@@ -45,4 +45,6 @@ suffix="
diff --git a/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft b/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft
new file mode 100644
index 0000000000..4ac0175212
--- /dev/null
+++ b/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft
@@ -0,0 +1,3 @@
+MARC XML
+Standard MARC XML output
+
@@ -38,6 +37,7 @@
Download Video
+
@@ -47,8 +47,5 @@
-
-
-
Found preformatted output for record %i (cache updated on %s).
"""
+BibFormat element: bookmark toolbar.
-import cgi
+Uses: .
+"""
-from invenio.config import CFG_SITE_URL, CFG_BASE_URL, CFG_SITE_RECORD, CFG_CERN_SITE
-from invenio.search_engine import record_public_p
-from invenio.htmlutils import escape_javascript_string
-from invenio.bibformat_elements.bfe_sciencewise import create_sciencewise_url, \
+from invenio.bibformat_elements.bfe_sciencewise import(
+ create_sciencewise_url,
get_arxiv_reportnumber
-from invenio.webjournal_utils import \
- parse_url_string, \
- make_journal_url, \
- get_journals_ids_and_names
-
-def format_element(bfo, only_public_records=1, sites="linkedin,twitter,facebook,google,delicious,sciencewise"):
+)
+from invenio.config import(
+ CFG_BASE_URL,
+ CFG_CERN_SITE,
+ CFG_SITE_RECORD,
+ CFG_SITE_URL
+)
+from invenio.htmlutils import(
+ escape_javascript_string
+)
+from invenio.search_engine import(
+ record_public_p
+)
+from invenio.webjournal_utils import(
+ get_journals_ids_and_names,
+ make_journal_url,
+ parse_url_string
+)
+
+
+def format_element(
+ bfo,
+ only_public_records=1,
+ sites="linkedin,twitter,facebook,google,delicious,sciencewise"
+):
"""
- Return a snippet of JavaScript needed for displaying a bookmark toolbar
+ Return a snippet of JavaScript needed for displaying a bookmark toolbar.
@param only_public_records: if set to 1 (the default), prints the box only
- if the record is public (i.e. if it belongs to the root colletion and is
- accessible to the world).
+ if the record is public (i.e. if it belongs to the root colletion and
+ is accessible to the world).
- @param sites: which sites to enable (default is 'linkedin,twitter,facebook,google,delicious,sciencewise'). This should be a
- comma separated list of strings.
+ @param sites: which sites to enable.
+ Default is 'linkedin,twitter,facebook,google,delicious,sciencewise').
+ This should be a comma separated list of strings.
Valid values are available on:
-
+ .
Note that 'sciencewise' is an ad-hoc service that will be displayed
only in case the record has an arXiv reportnumber and will always
be displayed last.
+ Note that "google_plusone" is an ad-hoc service. More information at:
+
+
+
+%(google_plusone_button)s
@@ -118,22 +158,35 @@ def format_element(bfo, only_public_records=1, sites="linkedin,twitter,facebook,
// ]]>
+%(google_plusone_script)s
""" % {
'siteurl': CFG_BASE_URL,
'sciencewise': sciencewise_script,
- 'title': escape_javascript_string(title,
- escape_for_html=False,
- escape_CDATA=True),
- 'description': escape_javascript_string(description,
- escape_for_html=False,
- escape_CDATA=True),
+ 'title': escape_javascript_string(
+ title,
+ escape_for_html=False,
+ escape_CDATA=True
+ ),
+ 'description': escape_javascript_string(
+ description,
+ escape_for_html=False,
+ escape_CDATA=True
+ ),
'sites_js': sites_js,
'url': url,
+ "google_plusone_button":
+ google_plusone_p and google_plusone_button or "",
+ "google_plusone_style":
+ google_plusone_p and google_plusone_style or "",
+ "google_plusone_script":
+ google_plusone_p and google_plusone_script or "",
}
+
def escape_values(bfo):
"""
- Called by BibFormat in order to check if output of this element
- should be escaped.
+ Called by BibFormat.
+
+ Checks if the output of this element should be escaped.
"""
return 0
diff --git a/modules/bibformat/lib/elements/bfe_copyright.py b/modules/bibformat/lib/elements/bfe_copyright.py
index aaa1be32c0..d738a27ef9 100644
--- a/modules/bibformat/lib/elements/bfe_copyright.py
+++ b/modules/bibformat/lib/elements/bfe_copyright.py
@@ -3,7 +3,8 @@
# $Id$
#
# This file is part of Invenio.
-# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2012, 2013, 2014 CERN.
+# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
+# 2012, 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -36,7 +37,7 @@
def format_element(bfo, copyrights_separator=", ", licenses_separator=", ", instances_separator=", ",
link_to_licenses='yes', auto_link_to_CERN_license='no', remove_link_to_CERN_license='yes',
- show_licenses='yes', show_material="yes",
+ show_licenses='yes', show_material="yes",
show_sponsor="yes", license_to_url_kb='LICENSE2URL'):
"""
Print copyright information
@@ -84,12 +85,19 @@ def format_element(bfo, copyrights_separator=", ", licenses_separator=", ", inst
if copyright_info.has_key('f'):
# Copyright message. Use this as label
label = copyright_info['f']
- elif copyright_info.has_key('d'):
+ elif 'd' in copyright_info:
# Copyright holder
year = ''
- if copyright_info.has_key('g'):
+ if 'g' in copyright_info:
# Year was given. Use it too
- year = "%s " % copyright_info['g']
+ # Also include the current year, if different
+ from datetime import date
+ current_year = date.today().year
+ if current_year > int(copyright_info['g']):
+ year_end = "-{0}".format(current_year)
+ else:
+ year_end = ""
+ year = "{0}{1} ".format(copyright_info['g'], year_end)
label = "© " + year + copyright_info['d']
if copyright_info['d'] == 'CERN' and \
len(licenses_info) == 0 and \
diff --git a/modules/bibformat/lib/elements/bfe_edit_record.py b/modules/bibformat/lib/elements/bfe_edit_record.py
index 3fe43c4f42..cbeba2fcb3 100644
--- a/modules/bibformat/lib/elements/bfe_edit_record.py
+++ b/modules/bibformat/lib/elements/bfe_edit_record.py
@@ -36,7 +36,7 @@ def format_element(bfo, style):
out = ""
user_info = bfo.user_info
- if user_can_edit_record_collection(user_info, bfo.recID):
+ if user_can_edit_record_collection(user_info, int(bfo.recID)):
linkattrd = {}
if style != '':
linkattrd['style'] = style
diff --git a/modules/bibformat/lib/elements/bfe_record_recommendations.py b/modules/bibformat/lib/elements/bfe_record_recommendations.py
new file mode 100644
index 0000000000..b7cc5a8d69
--- /dev/null
+++ b/modules/bibformat/lib/elements/bfe_record_recommendations.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2015, 2016 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""BibFormat element.
+
+* Creates a list of record recommendations
+"""
+
+import re
+
+from invenio.config import CFG_RECOMMENDER_REDIS
+from invenio.messages import gettext_set_language
+
+
+html_script = """
+
+"""
+
+
+def format_element(bfo):
+ """Create the HTML and JS code to display the recommended records."""
+ if CFG_RECOMMENDER_REDIS == "":
+ return ""
+ try:
+ re.search(r'/record/', bfo.user_info.get('uri')).group()
+ except AttributeError:
+ # No record url found
+ return ""
+
+ _ = gettext_set_language(bfo.lang)
+
+ url = "/record/" + str(bfo.recID) + "/recommendations"
+ html = html_script % {'recommendations_url': url,
+ 'text_title': _("You might also be interested in"),
+ }
+ return html
+
+
+def escape_values(bfo):
+ """Called by BibFormat to check if output should be escaped."""
+ return 0
diff --git a/modules/bibformat/lib/elements/bfe_record_stats.py b/modules/bibformat/lib/elements/bfe_record_stats.py
index e9da239a6f..bcd98ca16c 100644
--- a/modules/bibformat/lib/elements/bfe_record_stats.py
+++ b/modules/bibformat/lib/elements/bfe_record_stats.py
@@ -19,6 +19,23 @@
__revision__ = "$Id$"
from invenio.dbquery import run_sql
+ELASTICSEARCH_ENABLED = False
+
+try:
+ from elasticsearch import Elasticsearch
+ from invenio.config import \
+ CFG_ELASTICSEARCH_LOGGING, \
+ CFG_ELASTICSEARCH_SEARCH_HOST, \
+ CFG_ELASTICSEARCH_INDEX_PREFIX
+
+ # if we were able to import all modules and ES logging is enabled, then use
+ # elasticsearch instead of normal db queries
+ if CFG_ELASTICSEARCH_LOGGING:
+ ELASTICSEARCH_ENABLED = True
+except ImportError:
+ pass
+ # elasticsearch not supported
+
def format_element(bfo, display='day_distinct_ip_nb_views'):
'''
@@ -26,31 +43,212 @@ def format_element(bfo, display='day_distinct_ip_nb_views'):
@param display: the type of statistics displayed. Can be 'total_nb_view', 'day_nb_views', 'total_distinct_ip_nb_views', 'day_distincts_ip_nb_views', 'total_distinct_ip_per_day_nb_views'
'''
+ if ELASTICSEARCH_ENABLED:
+ page_views = 0
+ ES_INDEX = CFG_ELASTICSEARCH_INDEX_PREFIX + "*"
+ recID = bfo.recID
+ query = ""
+
+ es = Elasticsearch(CFG_ELASTICSEARCH_SEARCH_HOST)
+ if display == 'total_nb_views':
+ query = {
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "id_bibrec": recID
+ }
+ },
+ {
+ "match": {
+ "_type": "events.pageviews"
+ }
+ }
+ ]
+ }
+ }
+ }
+ results = es.count(index=ES_INDEX, body=query)
+ if results:
+ page_views = results.get('count', 0)
+ elif display == 'day_nb_views':
+ query = {
+ "query": {
+ "filtered": {
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "id_bibrec": recID
+ }
+ },
+ {
+ "match": {
+ "_type": "events.pageviews"
+ }
+ }
+ ]
+ }
+ },
+ "filter": {
+ "range": {
+ "@timestamp": {
+ "gt": "now-1d"
+ }
+ }
+ }
+ }
+ }
+ }
+ results = es.count(index=ES_INDEX, body=query)
+ if results:
+ page_views = results.get('count', 0)
+ elif display == 'total_distinct_ip_nb_views':
+ search_type = "count"
+ # TODO this search query with aggregation is slow, maybe there is a way to make it faster ?
+ query = {
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "id_bibrec": recID
+ }
+ },
+ {
+ "match": {
+ "_type": "events.pageviews"
+ }
+ }
+ ]
+ }
+ },
+ "aggregations": {
+ "distinct_ips": {
+ "cardinality": {
+ "field": "client_host"
+ }
+ }
+ }
+ }
+ results = es.search(index=ES_INDEX, body=query, search_type=search_type)
+ if results:
+ page_views = results.get('aggregations', {}).get('distinct_ips', {}).get('value', 0)
+ elif display == 'day_distinct_ip_nb_views':
+ search_type = "count"
+ # TODO aggregation is slow, maybe there is a way to make a faster query
+ query = {
+ "query": {
+ "filtered": {
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "id_bibrec": recID
+ }
+ },
+ {
+ "match": {
+ "_type": "events.pageviews"
+ }
+ }
+ ]
+ }
+ },
+ "filter": {
+ "range": {
+ "@timestamp": {
+ "gt": "now-1d"
+ }
+ }
+ }
+ }
+ },
+ "aggregations": {
+ "distinct_ips": {
+ "cardinality": {
+ "field": "client_host"
+ }
+ }
+ }
+ }
+ results = es.search(index=ES_INDEX, body=query, search_type=search_type)
+ if results:
+ page_views = results.get('aggregations', {}).get('distinct_ips', {}).get('value', 0)
+ elif display == 'total_distinct_ip_per_day_nb_views':
+ search_type = "count"
+ # TODO aggregation is slow, maybe there is a way to make a faster query
+ query = {
+ "query": {
+ "filtered": {
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "match": {
+ "id_bibrec": recID
+ }
+ },
+ {
+ "match": {
+ "_type": "events.pageviews"
+ }
+ }
+ ]
+ }
+ }
+ }
+ },
+ "aggregations": {
+ "daily_stats": {
+ "date_histogram": {
+ "field": "@timestamp",
+ "interval": "day"
+ },
+ "aggregations": {
+ "distinct_ips": {
+ "cardinality": {
+ "field": "client_host"
+ }
+ }
+ }
+ }
+ }
+ }
+ results = es.search(index=ES_INDEX, body=query, search_type=search_type)
+ if results:
+ buckets = results.get("aggregations", {}).get("daily_stats", {}).get("buckets", {})
+ page_views = sum([int(bucket.get("distinct_ips", {}).get('value', '0')) for bucket in buckets])
+ return page_views
+ else:
- if display == 'total_nb_views':
- return run_sql("""SELECT COUNT(client_host) FROM rnkPAGEVIEWS
- WHERE id_bibrec=%s""",
- (bfo.recID,))[0][0]
- elif display == 'day_nb_views':
- return run_sql("""SELECT COUNT(client_host) FROM rnkPAGEVIEWS
- WHERE id_bibrec=%s AND DATE(view_time)=CURDATE()""",
- (bfo.recID,))[0][0]
- elif display == 'total_distinct_ip_nb_views':
- return run_sql("""SELECT COUNT(DISTINCT client_host) FROM rnkPAGEVIEWS
- WHERE id_bibrec=%s""",
- (bfo.recID,))[0][0]
- elif display == 'day_distinct_ip_nb_views':
- return run_sql("""SELECT COUNT(DISTINCT client_host) FROM rnkPAGEVIEWS
- WHERE id_bibrec=%s AND DATE(view_time)=CURDATE()""",
- (bfo.recID,))[0][0]
- elif display == 'total_distinct_ip_per_day_nb_views':
- # Count the number of distinct IP addresses for every day Then
- # sum up. Similar to total_distinct_users_nb_views but assume
- # that several different users can be behind a single IP
- # (which could change every day)
- res = run_sql("""SELECT COUNT(DISTINCT client_host)
- FROM rnkPAGEVIEWS
- WHERE id_bibrec=%s GROUP BY DATE(view_time)""",
- (bfo.recID,))
- return sum([row[0] for row in res])
+ if display == 'total_nb_views':
+ return run_sql("""SELECT COUNT(client_host) FROM rnkPAGEVIEWS
+ WHERE id_bibrec=%s""",
+ (bfo.recID,))[0][0]
+ elif display == 'day_nb_views':
+ return run_sql("""SELECT COUNT(client_host) FROM rnkPAGEVIEWS
+ WHERE id_bibrec=%s AND DATE(view_time)=CURDATE()""",
+ (bfo.recID,))[0][0]
+ elif display == 'total_distinct_ip_nb_views':
+ return run_sql("""SELECT COUNT(DISTINCT client_host) FROM rnkPAGEVIEWS
+ WHERE id_bibrec=%s""",
+ (bfo.recID,))[0][0]
+ elif display == 'day_distinct_ip_nb_views':
+ return run_sql("""SELECT COUNT(DISTINCT client_host) FROM rnkPAGEVIEWS
+ WHERE id_bibrec=%s AND DATE(view_time)=CURDATE()""",
+ (bfo.recID,))[0][0]
+ elif display == 'total_distinct_ip_per_day_nb_views':
+ # Count the number of distinct IP addresses for every day Then
+ # sum up. Similar to total_distinct_users_nb_views but assume
+ # that several different users can be behind a single IP
+ # (which could change every day)
+ res = run_sql("""SELECT COUNT(DISTINCT client_host)
+ FROM rnkPAGEVIEWS
+ WHERE id_bibrec=%s GROUP BY DATE(view_time)""",
+ (bfo.recID,))
+ return sum([row[0] for row in res])
diff --git a/modules/bibformat/lib/elements/bfe_sword_push.py b/modules/bibformat/lib/elements/bfe_sword_push.py
index fd170eca37..76b3b659c7 100644
--- a/modules/bibformat/lib/elements/bfe_sword_push.py
+++ b/modules/bibformat/lib/elements/bfe_sword_push.py
@@ -38,14 +38,12 @@ def format_element(bfo, remote_server_id, link_label="Push via Sword"):
return ""
sword_arguments = {'ln': bfo.lang,
- 'recid': bfo.recID}
+ 'record_id': bfo.recID}
if remote_server_id:
- sword_arguments['id_remote_server'] = remote_server_id
- else:
- sword_arguments['status'] = 'select_server'
+ sword_arguments['server_id'] = remote_server_id
- return create_html_link(CFG_BASE_URL + '/bibsword',
+ return create_html_link(CFG_BASE_URL + '/sword_client/submit',
sword_arguments,
link_label)
diff --git a/modules/bibformat/lib/elements/bfe_xml_record.py b/modules/bibformat/lib/elements/bfe_xml_record.py
index 020987a902..e4c6c40d37 100644
--- a/modules/bibformat/lib/elements/bfe_xml_record.py
+++ b/modules/bibformat/lib/elements/bfe_xml_record.py
@@ -24,14 +24,27 @@ def format_element(bfo, type='xml', encodeForXML='yes'):
"""
Prints the complete current record as XML.
- @param type: the type of xml. Can be 'xml', 'oai_dc', 'marcxml', 'xd'
+ @param type: the type of xml. Can be 'xml', 'oai_dc', 'marcxml', 'xd', 'xmf'
@param encodeForXML: if 'yes', replace all < > and & with html corresponding escaped characters.
"""
from invenio.bibformat_utils import record_get_xml
+ from invenio.bibrecord import get_filtered_record, record_xml_output, get_marc_tag_extended_with_wildcards
from invenio.textutils import encode_for_xml
+ from invenio.search_engine import get_record
#Can be used to output various xml flavours.
- out = record_get_xml(bfo.recID, format=type, on_the_fly=True)
+ out = ''
+ if type == 'xmf':
+ tags = bfo.ot
+ if tags != ['']:
+ filter_tags = map(get_marc_tag_extended_with_wildcards, tags)
+ else:
+ filter_tags = []
+ record = get_record(bfo.recID)
+ filtered_record = get_filtered_record(record, filter_tags)
+ out = record_xml_output(filtered_record)
+ else:
+ out = record_get_xml(bfo.recID, format=type, on_the_fly=True)
if encodeForXML.lower() == 'yes':
return encode_for_xml(out)
diff --git a/modules/bibformat/lib/elements/bfe_youtube_authorization.py b/modules/bibformat/lib/elements/bfe_youtube_authorization.py
new file mode 100644
index 0000000000..985cd28c51
--- /dev/null
+++ b/modules/bibformat/lib/elements/bfe_youtube_authorization.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2007, 2008, 2009, 2010, 2011, 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""BibFormat element - Handle youtube authrization"""
+
+from invenio.bibencode_youtube import youtube_script
+from invenio.access_control_admin import acc_is_user_in_role, acc_get_role_id
+from invenio.bibencode_config import CFG_BIBENCODE_YOUTUBE_USER_ROLE
+
+def format_element(bfo):
+ """
+ Handles youtube authorization
+ =============================
+ """
+ if acc_is_user_in_role(bfo.user_info, acc_get_role_id(CFG_BIBENCODE_YOUTUBE_USER_ROLE)):
+ return youtube_script(bfo.recID)
+
+def escape_values(bfo):
+ """
+ Called by BibFormat in order to check if output of this element
+ should be escaped.
+ """
+ return 0
diff --git a/modules/bibindex/lib/bibindex_termcollectors.py b/modules/bibindex/lib/bibindex_termcollectors.py
index 464cc31cc5..823fc85350 100644
--- a/modules/bibindex/lib/bibindex_termcollectors.py
+++ b/modules/bibindex/lib/bibindex_termcollectors.py
@@ -158,6 +158,8 @@ def _get_phrases_for_tokenizing(self, tag, recIDs):
for recID in recIDs:
control_nos = get_fieldvalues(recID, authority_tag)
for control_no in control_nos:
+ if not control_no:
+ continue
new_strings = get_index_strings_by_control_no(control_no)
for string_value in new_strings:
phrases.add((recID, string_value))
diff --git a/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py b/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py
index c26dbdfbcb..3f2b04a582 100644
--- a/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py
+++ b/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py
@@ -43,6 +43,7 @@
from invenio.bibtask import write_message
from invenio.errorlib import register_exception
from invenio.intbitset import intbitset
+from invenio.search_engine import search_pattern
from invenio.bibindex_tokenizers.BibIndexDefaultTokenizer import BibIndexDefaultTokenizer
@@ -131,16 +132,18 @@ def get_words_from_fulltext(self, url_direct_or_indirect):
for splash_re, url_re in CFG_BIBINDEX_SPLASH_PAGES.iteritems():
if re.match(splash_re, url_direct_or_indirect):
write_message("... %s is a splash page (%s)" % (url_direct_or_indirect, splash_re), verbose=2)
- html = urllib2.urlopen(url_direct_or_indirect).read()
- urls = get_links_in_html_page(html)
- write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3)
- for url in urls:
- if re.match(url_re, url):
- write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2)
- urls_to_index.add(url)
- if not urls_to_index:
- urls_to_index.add(url_direct_or_indirect)
- write_message("... will extract words from %s" % ', '.join(urls_to_index), verbose=2)
+ if url_re is None:
+ urls_to_index.add(url_direct_or_indirect)
+ continue
+ else:
+ html = urllib2.urlopen(url_direct_or_indirect).read()
+ urls = get_links_in_html_page(html)
+ write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3)
+ for url in urls:
+ if re.match(url_re, url):
+ write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2)
+ urls_to_index.add(url)
+ write_message("... will extract words from {0}".format(urls_to_index), verbose=2)
words = {}
for url in urls_to_index:
tmpdoc = download_url(url)
@@ -156,11 +159,14 @@ def get_words_from_fulltext(self, url_direct_or_indirect):
indexer = get_idx_indexer('fulltext')
if indexer != 'native':
- if indexer == 'SOLR' and CFG_SOLR_URL:
- solr_add_fulltext(None, text) # FIXME: use real record ID
- if indexer == 'XAPIAN' and CFG_XAPIAN_ENABLED:
- #xapian_add(None, 'fulltext', text) # FIXME: use real record ID
- pass
+ recids = search_pattern(p='8567_u:{0}'.format(url))
+ write_message('... will add words to record {0}'.format(recid), verbose=2)
+ for recid in recids:
+ if indexer == 'SOLR' and CFG_SOLR_URL:
+ solr_add_fulltext(recid, text)
+ if indexer == 'XAPIAN' and CFG_XAPIAN_ENABLED:
+ #xapian_add(recid, 'fulltext', text)
+ pass
# we are relying on an external information retrieval system
# to provide full-text indexing, so dispatch text to it and
# return nothing here:
diff --git a/modules/bibrank/lib/bibrank_citation_searcher.py b/modules/bibrank/lib/bibrank_citation_searcher.py
index 31e81dec16..05235688fe 100644
--- a/modules/bibrank/lib/bibrank_citation_searcher.py
+++ b/modules/bibrank/lib/bibrank_citation_searcher.py
@@ -79,10 +79,10 @@ def cache_filler():
def timestamp_verifier():
citation_lastupdate = get_lastupdated('citation')
- if citation_lastupdate:
+ try:
return citation_lastupdate.strftime("%Y-%m-%d %H:%M:%S")
- else:
- return "0000-00-00 00:00:00"
+ except AttributeError:
+ return citation_lastupdate
DataCacher.__init__(self, cache_filler, timestamp_verifier)
diff --git a/modules/bibrank/lib/bibrank_downloads_similarity.py b/modules/bibrank/lib/bibrank_downloads_similarity.py
index d79ff4f33d..5921cd013b 100644
--- a/modules/bibrank/lib/bibrank_downloads_similarity.py
+++ b/modules/bibrank/lib/bibrank_downloads_similarity.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2012 CERN.
+# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2012, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -20,12 +20,14 @@
__revision__ = \
"$Id$"
-from invenio.config import \
- CFG_ACCESS_CONTROL_LEVEL_SITE, \
- CFG_CERN_SITE
+from invenio.config import (
+ CFG_ACCESS_CONTROL_LEVEL_SITE, CFG_CERN_SITE,
+ CFG_ELASTICSEARCH_BOT_AGENT_STRINGS
+)
from invenio.dbquery import run_sql
from invenio.bibrank_downloads_indexer import database_tuples_to_single_list
from invenio.search_engine_utils import get_fieldvalues
+from invenio.errorlib import register_exception
def record_exists(recID):
"""Return 1 if record RECID exists.
@@ -46,7 +48,7 @@ def record_exists(recID):
### INTERFACE
-def register_page_view_event(recid, uid, client_ip_address):
+def register_page_view_event(recid, uid, client_ip_address, user_agent):
"""Register Detailed record page view event for record RECID
consulted by user UID from machine CLIENT_HOST_IP.
To be called by the search engine.
@@ -55,10 +57,26 @@ def register_page_view_event(recid, uid, client_ip_address):
# do not register access if we are in read-only access control
# site mode:
return []
- return run_sql("INSERT INTO rnkPAGEVIEWS " \
- " (id_bibrec,id_user,client_host,view_time) " \
- " VALUES (%s,%s,INET_ATON(%s),NOW())", \
- (recid, uid, client_ip_address))
+
+ # register event in webstat
+ try:
+ from invenio.webstat import register_customevent
+ pageviews_register_event = [
+ recid, uid, client_ip_address, user_agent
+ ]
+ is_bot = False
+ if user_agent:
+ for bot in CFG_ELASTICSEARCH_BOT_AGENT_STRINGS:
+ if bot in user_agent:
+ is_bot = True
+ break
+ pageviews_register_event.append(is_bot)
+ register_customevent("pageviews", pageviews_register_event)
+ except:
+ register_exception(
+ ("Do the webstat tables exists? Try with 'webstatadmin"
+ " --load-config'")
+ )
def calculate_reading_similarity_list(recid, type="pageviews"):
"""Calculate reading similarity data to use in reading similarity
diff --git a/modules/bibrank/lib/bibrank_record_sorter.py b/modules/bibrank/lib/bibrank_record_sorter.py
index 05baa868d5..c9e71400ba 100644
--- a/modules/bibrank/lib/bibrank_record_sorter.py
+++ b/modules/bibrank/lib/bibrank_record_sorter.py
@@ -222,7 +222,7 @@ def citation(rank_method_code, related_to, hitset, rank_limit_relevance, verbose
return rank_by_citations(hits, verbose)
-def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[], verbose=0, field='', rg=None, jrec=None):
+def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[], verbose=0, field='', rg=None, jrec=None, kwargs={}):
"""Sorts given records or related records according to given method
Parameters:
@@ -293,7 +293,7 @@ def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[],
if function == "word_similarity_solr":
if verbose > 0:
voutput += "In Solr part:{0} '+result['error']+' '
- else:
- submissions = select_submitted_record_infos(0, 1, result['row_id'])
- if metadata['filename'] != '':
- upload_fulltext(recid, metadata['filename'])
-
- bibsword_template = BibSwordTemplate()
- body = bibsword_template.tmpl_display_list_submission(submissions)
+def _validate_server(server_settings):
+ """
+ Returs True if the server settings are valid or False if otherwise.
+ """
- return (body, errors, warnings)
+ (server_name,
+ server_engine,
+ server_username,
+ server_password,
+ server_email,
+ server_update_frequency) = server_settings
+
+ available_engines = _CLIENT_SERVERS.keys()
+ if "__init__" in available_engines:
+ available_engines.remove("__init__")
+ if server_engine not in available_engines:
+ return False
+
+ if not re.match(
+ r"^([0-9]+[wdhms]{1}){1,5}$",
+ server_update_frequency
+ ):
+ return False
+
+ return True
diff --git a/modules/bibsword/lib/bibsword_client_dblayer.py b/modules/bibsword/lib/bibsword_client_dblayer.py
index f52dc74fdf..10d8dd117b 100644
--- a/modules/bibsword/lib/bibsword_client_dblayer.py
+++ b/modules/bibsword/lib/bibsword_client_dblayer.py
@@ -1,5 +1,7 @@
+# -*- coding: utf-8 -*-
+#
# This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -15,349 +17,355 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-'''
-BibSWORD Client DBLayer
-'''
+"""
+BibSword Client DBLayer.
+"""
-import datetime
import time
+from invenio.dateutils import convert_datestruct_to_datetext
from invenio.dbquery import run_sql
-from invenio.bibsword_config import CFG_SUBMISSION_STATUS_PUBLISHED, \
- CFG_SUBMISSION_STATUS_REMOVED
-def get_remote_server_auth(id_remoteserver):
- '''
- This function select the username and the password stored in the
- table swrREMOTESERVER to execute HTTP Request
- @param id_remoteserver: id of the remote server to contact
- @return: (authentication_info) dictionnary conating username - password
- '''
-
- authentication_info = {'error':'',
- 'hostname':'',
- 'username':'',
- 'password':'',
- 'realm':'',
- 'url_servicedocument':''}
-
- qstr = '''SELECT host, username, password, realm, url_servicedocument ''' \
- ''' FROM swrREMOTESERVER WHERE id=%s'''
- qres = run_sql(qstr, (id_remoteserver, ))
-
- if len(qres) == 0 :
- authentication_info['error'] = '''The server id doesn't correspond ''' \
- '''to any remote server'''
- return authentication_info
+def store_temp_submission(
+ sid,
+ sobject_blob
+):
+ """
+ Store the temporary submission.
+ """
+
+ query = """
+ INSERT INTO swrCLIENTTEMPSUBMISSION
+ (id, object, last_updated)
+ VALUES (%s, %s, %s)
+ """
- (host, username, password, realm, url_servicedocument) = qres[0]
+ now = convert_datestruct_to_datetext(time.localtime())
+ params = (sid, sobject_blob, now)
- authentication_info['hostname'] = host
- authentication_info['username'] = username
- authentication_info['password'] = password
- authentication_info['realm'] = realm
- authentication_info['url_servicedocument'] = url_servicedocument
-
- return authentication_info
-
-
-def update_servicedocument(xml_servicedocument, id_remoteserver):
- '''
- This function update the servicedocument filed containing all the
- collections and categories for the given remote server
- @param xml_servicedocument: xml file
- @param id_remoteserver: id number of the remote server to update
- @return: (boolean) true if update successfull, false else
- '''
-
- # get the current time to keep the last update time
- current_type = datetime.datetime.now()
- formatted_current_time = time.mktime(current_type.timetuple())
-
- qstr = '''UPDATE swrREMOTESERVER ''' \
- '''SET xml_servicedocument=%s, last_update=%s ''' \
- '''WHERE id=%s'''
- qres = run_sql(qstr, (xml_servicedocument, formatted_current_time,
- id_remoteserver, ))
-
- return qres
-
-
-def select_servicedocument(id_remoteserver):
- '''
- This function retreive the servicedocument of the given remote server
- @param id_remoteserver: id number of the remote server selected
- @return: (xml_file) servicedocument xml file that contains coll and cat
- '''
-
- qstr = '''SELECT xml_servicedocument ''' \
- '''FROM swrREMOTESERVER ''' \
- '''WHERE id=%s'''
- qres = run_sql(qstr, (id_remoteserver, ))
-
- if len(qres) == 0 :
- return ''
-
- return qres[0][0]
-
-
-def get_last_update(id_remoteserver):
- '''
- This function return the last update time of the service document. This
- is usefull to know if the service collection needs to be refreshed
- @param id_remoteserver: id number of the remote server to check
- @return: (datetime) datetime of the last update (yyyy-mm-dd hh:mm:ss)
- '''
-
- qstr = '''SELECT last_update ''' \
- '''FROM swrREMOTESERVER ''' \
- '''WHERE id=%s '''
- qres = run_sql(qstr, (id_remoteserver, ))
-
- if len(qres) == 0:
- return '0'
-
- return qres[0][0]
-
-
-def get_all_remote_server(id_server):
- '''
- This function select the name of all remote service implementing the
- SWORD protocol. It returns a list of dictionnary containing three fields:
- id, name and host
- @return: (remote_server) list of dictionnary (id - name - host) of each
- remote server
- '''
-
- remote_servers = []
-
- if id_server == '':
- qstr = '''SELECT id, name, host FROM swrREMOTESERVER'''
- qres = run_sql(qstr)
- else :
- qstr = ''' SELECT id, name, host FROM swrREMOTESERVER WHERE id=%s'''
- qres = run_sql(qstr, (id_server, ))
+ try:
+ res = run_sql(query, params)
+ except Exception:
+ return False
+ else:
+ return True
- for res in qres:
- remote_server = {}
- remote_server['id'] = res[0]
- remote_server['name'] = res[1]
- remote_server['host'] = res[2]
- remote_servers.append(remote_server)
+def retrieve_temp_submission(
+ sid
+):
+ """
+ Retrieve the temporary submission.
+ """
- return remote_servers
+ query = """
+ SELECT object
+ FROM swrCLIENTTEMPSUBMISSION
+ WHERE id=%s
+ """
+ params = (sid,)
-def is_record_sent_to_server(id_server, id_record):
- '''
- check in the table swrCLIENTDATA that the current record has not already
- been sent before
- @param id_server: id of the remote server where to send the record
- @param id_record: id of the record to send
- return : True if a value was found, false else
- '''
+ res = run_sql(query, params)
- qstr = '''SELECT COUNT(*) FROM swrCLIENTDATA ''' \
- '''WHERE id_swrREMOTESERVER=%s AND id_record=%s ''' \
- '''AND status NOT LIKE 'removed' '''
- qres = run_sql(qstr, (id_server, id_record, ))
-
- if (qres[0][0] == 0):
+ if res:
+ return res[0][0]
+ else:
+ return None
+
+
+def update_temp_submission(
+ sid,
+ sobject_blob
+):
+ """
+ Update the temporary submission.
+ """
+
+ query = """
+ UPDATE swrCLIENTTEMPSUBMISSION
+ SET object=%s,
+ last_updated=%s
+ WHERE id=%s
+ """
+
+ now = convert_datestruct_to_datetext(time.localtime())
+ params = (sobject_blob, now, sid)
+
+ res = run_sql(query, params)
+
+ return res
+
+
+def delete_temp_submission(
+ sid
+):
+ """
+ Delete the temporary submission.
+ """
+
+ query = """
+ DELETE FROM swrCLIENTTEMPSUBMISSION
+ WHERE id=%s
+ """
+
+ params = (sid,)
+
+ res = run_sql(query, params)
+
+ return res
+
+
+def archive_submission(
+ user_id,
+ record_id,
+ server_id,
+ alternate_url,
+ edit_url
+):
+ """
+ Archive the given submission.
+ """
+
+ query = """
+ INSERT INTO swrCLIENTSUBMISSION
+ (
+ id_user,
+ id_record,
+ id_server,
+ url_alternate,
+ url_edit,
+ submitted
+ )
+ VALUES (
+ %s,
+ %s,
+ %s,
+ %s,
+ %s,
+ %s
+ )
+ """
+
+ now = convert_datestruct_to_datetext(time.localtime())
+ params = (
+ user_id,
+ record_id,
+ server_id,
+ alternate_url,
+ edit_url,
+ now
+ )
+
+ try:
+ res = run_sql(query, params)
+ except Exception:
return False
else:
return True
-def insert_into_swr_clientdata(id_swr_remoteserver,
- recid,
- report_no,
- remote_id,
- id_user,
- user_name,
- user_email,
- xml_media_deposit,
- xml_metadata_submit,
- link_media,
- link_metadata,
- link_status):
- '''
- This method insert a new row in the swrCLIENTDATA table. Some are given in
- parameters and some other such as the insertion time and the submission
- status are set by default
- @param id_swr_remoteserver: foreign key of the sword remote server
- @param recid: foreign key of the submitted record
- @param id_user: foreign key of the user who did the submission
- @param xml_media_deposit: xml response after the media deposit
- @param xml_metadata_submit: xml response after the metadata submission
- @param remote_id: record id given by the remote server in the response
- @param link_media: remote url where to find the depositted medias
- @param link_metadata: remote url where to find the submitted metadata
- @param link_status: remote url where to check the submission status
- @return: (result) the id of the new row inserted, 0 if the submission
- didn't work
- '''
-
- current_date = time.strftime("%Y-%m-%d %H:%M:%S")
- xml_media_deposit.replace("\"", "\'")
- xml_metadata_submit.encode('utf-8')
-
- qstr = '''INSERT INTO swrCLIENTDATA (id_swrREMOTESERVER, id_record, ''' \
- '''report_no, id_remote, id_user, user_name, user_email, ''' \
- '''xml_media_deposit, xml_metadata_submit, submission_date, ''' \
- '''link_medias, link_metadata, link_status, last_update) ''' \
- '''VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ''' \
- '''%s, %s) '''
- qres = run_sql(qstr, (id_swr_remoteserver, recid, report_no, remote_id,
- id_user, user_name, user_email, xml_media_deposit,
- xml_metadata_submit, current_date, link_media,
- link_metadata, link_status, current_date, ))
- return qres
-
-
-def count_nb_submitted_record() :
- '''
- return : the amount of submitted records
- '''
-
- qstr = '''SELECT COUNT(*) FROM swrCLIENTDATA'''
- qres = run_sql(qstr, ())
-
- return qres[0][0]
-
-
-def delete_from_swr_clientdata(id_submit):
- '''
- delete the given row from the swrCLIENTDATA table. Used by the test suit
- @param id_submit: id of the row to delete
- result : boolean, true if deleted, false else
- '''
-
- qstr = ''' DELETE FROM swrCLIENTDATA WHERE id=%s '''
- qres = run_sql(qstr, (id_submit, ))
-
- return qres
-
-
-def select_submitted_record_infos(first_row=0, offset=10, row_id=''):
- '''
- this method return a bidimentionnal table containing all rows of the
- table swrCLIENTDATA. If sepecified, the search can be limited to a
- server, a record or a record on a server
- @param first_row: give the limit where to start the selection
- @param offset: give the maximal amount of rows to select
- @return: table of row containing each colomn of the table swrCLIENTDATA
-
- FIXME: first_row is apparently supposed to select the chosen
- id_swrREMOTESERVER, but it is currently strangely handled...
- '''
-
- wstr = ''
- if row_id != '' :
- wstr = '''WHERE d.id = %s '''
- qstr = '''SELECT d.id, d.id_swrREMOTESERVER, r.name, r.host , ''' \
- '''d.id_record, d.report_no, d.id_remote, d.id_user, ''' \
- '''d.user_name, d.user_email, d.submission_date, ''' \
- '''d.publication_date, d.removal_date, d.link_medias, ''' \
- '''d.link_metadata, d.link_status, d.status, r.url_base_record ''' \
- '''FROM swrCLIENTDATA as d inner join swrREMOTESERVER ''' \
- '''as r ON d.id_swrREMOTESERVER = r.id ''' + wstr + \
- '''ORDER BY d.last_update DESC LIMIT %s,%s'''
- if wstr != '' :
- qres = run_sql(qstr, (row_id, first_row, offset, ))
- else :
- qres = run_sql(qstr, (first_row, offset, ))
-
- results = []
- for res in qres :
- result = {'publication_date':'', 'removal_date':''}
- result['id'] = res[0]
- result['id_server'] = res[1]
- result['server_name'] = res[2]
- result['server_host'] = res[3]
- result['id_record'] = res[4]
- result['report_no'] = res[5]
- result['id_remote'] = res[6]
- result['id_user'] = res[7]
- result['user_name'] = res[8]
- result['user_email'] = res[9]
- result['submission_date'] = res[10].strftime("%Y-%m-%d %H:%M:%S")
- if res[11] != None :
- result['publication_date'] = res[11].strftime("%Y-%m-%d %H:%M:%S")
- if res[12] != None :
- result['removal_date'] = res[12].strftime("%Y-%m-%d %H:%M:%S")
- result['link_medias'] = res[13]
- result['link_metadata'] = res[14]
- result['link_status'] = res[15]
- result['status'] = res[16]
- result['url_base_remote'] = res[17]
- results.append(result)
-
- return results
-
-
-def update_submission_status(id_record, status, remote_id=''):
- '''
- update the submission field with the new status of the submission
- @param id_record: id of the row to update
- @param status: new value to set in the status field
- @return: true if update done, else, false
- '''
-
- current_date = time.strftime("%Y-%m-%d %H:%M:%S")
-
- if status == CFG_SUBMISSION_STATUS_PUBLISHED and remote_id != '' :
- qstr = '''UPDATE swrCLIENTDATA SET status=%s, id_remote=%s, ''' \
- '''publication_date=%s, last_update=%s WHERE id=%s '''
- qres = run_sql(qstr, (status, remote_id, current_date, current_date,
- id_record, ))
-
-
- if status == CFG_SUBMISSION_STATUS_REMOVED :
- qstr = '''UPDATE swrCLIENTDATA SET status=%s, removal_date=%s, ''' \
- '''last_update=%s WHERE id=%s '''
- qres = run_sql(qstr, (status, current_date, current_date, id_record, ))
-
- else :
- qstr = '''UPDATE swrCLIENTDATA SET status=%s, last_update=%s ''' \
- '''WHERE id=%s '''
- qres = run_sql(qstr, (status, current_date, id_record, ))
-
- return qres
-
-
-def select_remote_server_infos(id_server):
- '''
- Select fields of the given remote server and return it in a tuple
- @param id_server: id of the server to select
- @return: (server_info) tuple containing all the available infos
- '''
-
- server_info = {'server_id' : '',
- 'server_name' : '',
- 'server_host' : '',
- 'username' : '',
- 'password' : '',
- 'email' : '',
- 'realm' : '',
- 'url_base_record' : '',
- 'url_servicedocument' : ''}
-
- qstr = '''SELECT id, name, host, username, password, email, realm, ''' \
- '''url_base_record, url_servicedocument ''' \
- '''FROM swrREMOTESERVER WHERE id = %s '''
- qres = run_sql(qstr, (id_server, ))
-
- result = qres[0]
-
- server_info['server_id'] = result[0]
- server_info['server_name'] = result[1]
- server_info['server_host'] = result[2]
- server_info['username'] = result[3]
- server_info['password'] = result[4]
- server_info['email'] = result[5]
- server_info['realm'] = result[6]
- server_info['url_base_record'] = result[7]
- server_info['url_servicedocument'] = result[8]
-
- return server_info
+def is_submission_archived(
+ record_id,
+ server_id,
+):
+ """
+ If the given record has already been archived to the given server
+ return True, otherwise return False.
+ """
+
+ query = """
+ SELECT COUNT(id_record)
+ FROM swrCLIENTSUBMISSION
+ WHERE id_record=%s
+ AND id_server=%s
+ """
+
+ params = (
+ record_id,
+ server_id,
+ )
+
+ res = run_sql(query, params)
+
+ return bool(res[0][0])
+
+
+def get_submissions(
+ with_dict=False
+):
+ """
+ Get the Sword client submissions.
+ """
+
+ query = """
+ SELECT user.nickname,
+ swrCLIENTSUBMISSION.id_record,
+ swrCLIENTSUBMISSION.id_server,
+ swrCLIENTSERVER.name,
+ swrCLIENTSUBMISSION.submitted,
+ swrCLIENTSUBMISSION.status,
+ swrCLIENTSUBMISSION.last_updated,
+ swrCLIENTSUBMISSION.url_alternate
+ FROM swrCLIENTSUBMISSION
+ JOIN swrCLIENTSERVER
+ ON swrCLIENTSUBMISSION.id_server = swrCLIENTSERVER.id
+ JOIN user
+ ON swrCLIENTSUBMISSION.id_user = user.id
+ ORDER BY swrCLIENTSUBMISSION.submitted DESC
+ """
+
+ params = None
+
+ result = run_sql(query, params, with_dict=with_dict)
+
+ return result
+
+
+def get_servers(
+ server_id=None,
+ with_dict=False
+):
+ """
+ Get the Sword client servers.
+
+ Given a server_id, get that server only.
+ """
+
+ query = """
+ SELECT id AS server_id,
+ name,
+ engine,
+ username,
+ password,
+ email,
+ service_document_parsed,
+ update_frequency,
+ last_updated
+ FROM swrCLIENTSERVER
+ """
+
+ if server_id is not None:
+ query += """
+ WHERE id=%s
+ """
+ params = (server_id,)
+ else:
+ params = None
+
+ result = run_sql(query, params, with_dict=with_dict)
+
+ return result
+
+
+def delete_servers(
+ server_id=None
+):
+ """
+ Delete the Sword client servers.
+
+ Given a server_id, delete that server only.
+ """
+
+ query = """
+ DELETE
+ FROM swrCLIENTSERVER
+ """
+
+ if server_id is not None:
+ query += """
+ WHERE id=%s
+ """
+ params = (server_id,)
+ else:
+ params = None
+
+ result = run_sql(query, params)
+
+ return result
+
+
+def modify_server(
+ server_id,
+ server_name,
+ server_engine,
+ server_username,
+ server_password,
+ server_email,
+ server_update_frequency
+):
+ """
+ Modify the Sword client server.
+
+ Given a server_id.
+ """
+
+ query = """
+ UPDATE swrCLIENTSERVER
+ SET name=%s,
+ engine=%s,
+ username=%s,
+ password=%s,
+ email=%s,
+ update_frequency=%s
+ WHERE id=%s
+ """
+
+ params = (
+ server_name,
+ server_engine,
+ server_username,
+ server_password,
+ server_email,
+ server_update_frequency,
+ server_id,
+ )
+
+ result = run_sql(query, params)
+
+ return result
+
+
+def add_server(
+ server_name,
+ server_engine,
+ server_username,
+ server_password,
+ server_email,
+ server_update_frequency
+):
+ """
+ Add a Sword client server based on the given information.
+ """
+
+ query = """
+ INSERT INTO swrCLIENTSERVER
+ (name,
+ engine,
+ username,
+ password,
+ email,
+ update_frequency)
+ VALUES (%s,
+ %s,
+ %s,
+ %s,
+ %s,
+ %s)
+ """
+
+ params = (
+ server_name,
+ server_engine,
+ server_username,
+ server_password,
+ server_email,
+ server_update_frequency,
+ )
+
+ try:
+ result = run_sql(query, params)
+ except:
+ result = None
+
+ return result
diff --git a/modules/bibsword/lib/bibsword_client_formatter.py b/modules/bibsword/lib/bibsword_client_formatter.py
deleted file mode 100644
index 1f10c94426..0000000000
--- a/modules/bibsword/lib/bibsword_client_formatter.py
+++ /dev/null
@@ -1,1196 +0,0 @@
-##This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
-#
-# Invenio is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-#
-# Invenio is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Invenio; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
-'''
-BibSWORD Client Formatter
-'''
-
-import zipfile
-import os
-from tempfile import mkstemp
-from xml.dom import minidom
-from invenio.config import CFG_TMPDIR
-from invenio.bibtask import task_low_level_submission
-from invenio.bibsword_config import CFG_MARC_REPORT_NUMBER, \
- CFG_MARC_TITLE, \
- CFG_MARC_AUTHOR_NAME, \
- CFG_MARC_AUTHOR_AFFILIATION, \
- CFG_MARC_CONTRIBUTOR_NAME, \
- CFG_MARC_CONTRIBUTOR_AFFILIATION, \
- CFG_MARC_ABSTRACT, \
- CFG_MARC_ADDITIONAL_REPORT_NUMBER, \
- CFG_MARC_DOI, \
- CFG_MARC_JOURNAL_REF_CODE, \
- CFG_MARC_JOURNAL_REF_TITLE, \
- CFG_MARC_JOURNAL_REF_PAGE, \
- CFG_MARC_JOURNAL_REF_YEAR, \
- CFG_MARC_COMMENT, \
- CFG_MARC_RECORD_SUBMIT_INFO, \
- CFG_SUBMIT_ARXIV_INFO_MESSAGE, \
- CFG_DOCTYPE_UPLOAD_COLLECTION, \
- CFG_SUBMISSION_STATUS_SUBMITTED, \
- CFG_SUBMISSION_STATUS_PUBLISHED, \
- CFG_SUBMISSION_STATUS_ONHOLD, \
- CFG_SUBMISSION_STATUS_REMOVED
-from invenio.bibdocfile import BibRecDocs
-from invenio.bibformat_engine import BibFormatObject
-
-#-------------------------------------------------------------------------------
-# Formating servicedocument file
-#-------------------------------------------------------------------------------
-
-def format_remote_server_infos(servicedocument):
- '''
- Get all informations about the server's options such as SWORD version,
- maxUploadSize, ... These informations are found in the servicedocument
- of the given server
- @param servicedocument: xml servicedocument in a string format
- @return: server_infomation. tuple containing the version, the
- maxUploadSize and the available modes
- '''
-
- #contains information tuple {'version', 'maxUploadSize', 'verbose', 'noOp'}
- server_informations = {'version' : '',
- 'maxUploadSize' : '',
- 'verbose' : '',
- 'noOp' : '',
- 'error' : '' }
-
- # now the xml node are accessible by programation
- try:
- parsed_xml_collections = minidom.parseString(servicedocument)
- except IOError:
- server_informations['error'] = \
- 'No servicedocument found for the remote server'
- return server_informations
-
- # access to the root of the xml file
- xml_services = parsed_xml_collections.getElementsByTagName('service')
- xml_service = xml_services[0]
-
- # get value of the node
- version_node = xml_service.getElementsByTagName('sword:version')[0]
- server_informations['version'] = \
- version_node.firstChild.nodeValue.encode('utf-8')
-
- # get value of the node
- max_upload_node = xml_service.getElementsByTagName('sword:maxUploadSize')[0]
- server_informations['maxUploadSize'] = \
- max_upload_node.firstChild.nodeValue.encode('utf-8')
-
-
- # get value of the node
- verbose_node = xml_service.getElementsByTagName('sword:verbose')[0]
- server_informations['verbose'] = \
- verbose_node.firstChild.nodeValue.encode('utf-8')
-
- # get value of the node
- no_op_node = xml_service.getElementsByTagName('sword:noOp')[0]
- server_informations['noOp'] = \
- no_op_node.firstChild.nodeValue.encode('utf-8')
-
- return server_informations
-
-
-def format_remote_collection(servicedocument):
- '''
- The function parse the servicedocument document and return a list with
- the collections of the given file ['id', 'name', 'url']
- @param servicedocument: xml file returned by the remote server.
- @return: the list of collection found in the service document
- '''
-
- collections = [] # contains list of collection tuple {'id', 'url', 'label'}
-
- # get the collections root node
- collection_nodes = parse_xml_servicedocument_file(servicedocument)
-
- # i will be the id of the collection
- i = 1
-
- #---------------------------------------------------------------------------
- # recuperation of the collections
- #---------------------------------------------------------------------------
-
- # loop that goes in each node's collection of the document
- for collection_node in collection_nodes:
-
- # dictionnary that contains the collections
- collection = {}
-
- collection['id'] = str(i)
- i = i + 1
-
- # collection uri (where to deposit the media)
- collection['url'] = \
- collection_node.attributes['href'].value.encode('utf-8')
-
- # collection name that is displayed to the user
- xml_title = collection_node.getElementsByTagName('atom:title')
- collection['label'] = xml_title[0].firstChild.nodeValue.encode('utf-8')
-
- # collection added to the collections list
- collections.append(collection)
-
- return collections
-
-
-def format_collection_informations(servicedocument, id_collection):
- '''
- This methode parse the given servicedocument to find the given collection
- node. Then it retrieve all information about the collection that contains
- the collection node.
- @param servicedocument: xml file returned by the remote server.
- @param id_collection: position of the collection in the sd (1 = first)
- @return: (collection_informations) tuple containing infos
- '''
-
- # contains information tuple {[accept], 'collectionPolicy', 'mediation',
- # 'treatment', 'accept_packaging'}
- collection_informations = {}
-
- # get the collections root node
- collection_nodes = parse_xml_servicedocument_file(servicedocument)
-
- # recuperation of the selected collection
- collection_node = collection_nodes[int(id_collection)-1]
-
- # get value of the nodes
- accept_nodes = collection_node.getElementsByTagName('accept')
- accept = []
- for accept_node in accept_nodes:
- accept.append(accept_node.firstChild.nodeValue.encode('utf-8'))
-
- collection_informations['accept'] = accept
-
- # get value of the nodes
- collection_policy = \
- collection_node.getElementsByTagName('sword:collectionPolicy')[0]
- collection_informations['collectionPolicy'] = \
- collection_policy.firstChild.nodeValue.encode('utf-8')
-
- # get value of the nodes
- mediation = collection_node.getElementsByTagName('sword:mediation')[0]
- collection_informations['mediation'] = \
- mediation.firstChild.nodeValue.encode('utf-8')
-
- # get value of the nodes
- treatment = collection_node.getElementsByTagName('sword:treatment')[0]
- collection_informations['treatment'] = \
- treatment.firstChild.nodeValue.encode('utf-8')
-
- # get value of the nodes
- accept_packaging = \
- collection_node.getElementsByTagName('sword:acceptPackaging')[0]
- collection_informations['accept_packaging'] = \
- accept_packaging.firstChild.nodeValue.encode('utf-8')
-
- return collection_informations
-
-
-def format_primary_categories(servicedocument, collection_id=0):
- '''
- This method parse the servicedocument to retrieve the primary category
- of the given collection. If no collection is given, it takes the first
- one.
- @param servicedocument: xml file returned by the remote server.
- @param collection_id: id of the collection to search
- @return: list of primary categories tuple ('id', 'url', 'label')
- '''
-
- categories = [] # contains list of category tuple {'id', 'url', 'label'}
-
- # get the collections root node
- collection_nodes = parse_xml_servicedocument_file(servicedocument)
-
- # i will be the id of the collection
- i = 1
-
- # recuperation of the selected collection
- collection_node = collection_nodes[int(collection_id)-1]
-
- #---------------------------------------------------------------------------
- # recuperation of the categories
- #---------------------------------------------------------------------------
-
- # select all primary category nodes
- primary_categories_node = \
- collection_node.getElementsByTagName('arxiv:primary_categories')[0]
- primary_category_nodes = \
- primary_categories_node.getElementsByTagName('arxiv:primary_category')
-
- # loop that goes in each primary_category nodes
- for primary_category_node in primary_category_nodes:
-
- # dictionnary that contains the categories
- category = {}
-
- category['id'] = str(i)
- i = i + 1
-
- category['url'] = \
- primary_category_node.attributes['term'].value.encode('utf-8')
- category['label'] = \
- primary_category_node.attributes['label'].value.encode('utf-8')
-
- categories.append(category)
-
- return categories
-
-
-def format_secondary_categories(servicedocument, collection_id=0):
- '''
- This method parse the servicedocument to retrieve the optional categories
- of the given collection. If no collection is given, it takes the first
- one.
- @param servicedocument: xml file returned by the remote server.
- @param collection_id: id of the collection to search
- @return: list of optional categories tuple ('id', 'url', 'label')
- '''
-
- categories = [] # contains list of category tuple {'id', 'url', 'label'}
-
- # get the collections root node
- collection_nodes = parse_xml_servicedocument_file(servicedocument)
-
- # i will be the id of the collection
- i = 1
-
- # recuperation of the selected collection
- collection_id = int(collection_id) - 1
- collection_node = collection_nodes[int(collection_id)]
-
- #---------------------------------------------------------------------------
- # recuperation of the categories
- #---------------------------------------------------------------------------
-
- # select all primary category nodes
- categories_node = collection_node.getElementsByTagName('categories')[0]
- category_nodes = categories_node.getElementsByTagName('category')
-
- # loop that goes in each primary_category nodes
- for category_node in category_nodes:
-
- # dictionnary that contains the categories
- category = {}
-
- category['id'] = str(i)
- i = i + 1
-
- category['url'] = category_node.attributes['term'].value.encode('utf-8')
- category['label'] = \
- category_node.attributes['label'].value.encode('utf-8')
-
- categories.append(category)
-
- return categories
-
-
-def parse_xml_servicedocument_file(servicedocument):
- '''
- This method parse a string containing a servicedocument to retrieve the
- collection node. It is used by all function that needs to work with
- collections
- @param servicedocument: xml file in containing in a string
- @return: (collecion_node) root node of all collecions
- '''
-
- # now the xml node are accessible by programation
- parsed_xml_collections = minidom.parseString(servicedocument)
-
- # access to the root of the xml file
- xml_services = parsed_xml_collections.getElementsByTagName('service')
- xml_service = xml_services[0]
-
- # their is only the global workspace in this xml document
- xml_workspaces = xml_service.getElementsByTagName('workspace')
- xml_workspace = xml_workspaces[0]
-
- # contains all collections in the xml file
- collection_nodes = xml_workspace.getElementsByTagName('collection')
-
- return collection_nodes
-
-
-#-------------------------------------------------------------------------------
-# Formating marcxml file
-#-------------------------------------------------------------------------------
-
-def get_report_number_from_macrxml(marcxml):
- '''
- retrieve the record id stored in the marcxml file. The record is in the
- tag 'RECORD ID'
- @param marcxml: marcxml file where to look for the record id
- @return: the record id in a string
- '''
-
- #get the reportnumber tag list
- tag = CFG_MARC_REPORT_NUMBER
- if tag == '':
- return ''
-
- #variable that contains the result of the parsing of the marcxml file
- datafields = get_list_of_marcxml_datafields(marcxml)
-
- for datafield in datafields:
-
- report_number = get_subfield_value_from_datafield(datafield, tag)
- if report_number != '':
- return report_number
-
- return ''
-
-
-def get_medias_to_submit(media_paths):
- '''
- This method get a list of recod of submission. It format a list of
- media containing name, size, type and file for each media id
- @param media_paths: list of path to the media to upload
- @return: list of media tuple
- '''
-
- # define the return value
- media = {}
-
- fp = open("/tmp/test.txt", "w")
- fp.write(media_paths[0])
-
- if len(media_paths) > 1:
- media_paths = format_file_to_zip_archiv(media_paths)
- else:
- media_paths = media_paths[0]
-
- if media_paths != '':
- media['file'] = open(media_paths, "r").read()
- media['size'] = len(media['file'])
- media['name'] = media_paths.split('/')[-1].split(';')[0]
- media['type'] = 'application/%s' % media['name'].split('.')[-1]
-
- return media
-
-
-def get_media_from_recid(recid):
- '''
- This method get the file in the given url
- @param recid: id of the file to get
- '''
-
- medias = []
-
- bibarchiv = BibRecDocs(recid)
- bibdocs = bibarchiv.list_latest_files()
-
- for bibdocfile in bibdocs:
-
- bibfile = {'name': bibdocfile.get_full_name(),
- 'file': '',
- 'type': 'application/%s' % \
- bibdocfile.get_superformat().split(".")[-1],
- 'path': bibdocfile.get_full_path(),
- 'collection': bibdocfile.get_type(),
- 'size': bibdocfile.get_size(),
- 'loaded': False,
- 'selected': ''}
-
- if bibfile['collection'] == "Main":
- bibfile['selected'] = 'checked=yes'
-
- medias.append(bibfile)
-
- return medias
-
-
-def format_author_from_marcxml(marcxml):
- '''
- This method parse the marcxml file to retrieve the author of a document
- @param marcxml: the xml file to parse
- @return: tuple containing {'name', 'email' and 'affiliations'}
- '''
-
- #get the tag id for the given field
- main_author = CFG_MARC_AUTHOR_NAME
- main_author_affiliation = CFG_MARC_AUTHOR_AFFILIATION
-
- #variable that contains the result of the parsing of the marcxml file
- datafields = get_list_of_marcxml_datafields(marcxml)
-
- #init the author tuple
- author = {'name':'', 'email':'', 'affiliation':[]}
-
- for datafield in datafields:
-
- # retreive the main author
- if author['name'] == '':
- name = get_subfield_value_from_datafield(datafield, main_author)
- if name != '':
- author['name'] = name
-
- affiliation = get_subfield_value_from_datafield(datafield, main_author_affiliation)
- if affiliation != '':
- author['affiliation'].append(affiliation)
-
- return author
-
-
-def format_marcxml_file(marcxml, is_file=False):
- '''
- Parse the given marcxml file to retreive the metadata needed by the
- forward of the document to ArXiv.org
- @param marcxml: marxml file that contains metadata from Invenio
- @return: (dictionnary) couple of key value needed for the push
- '''
-
- #init the return tuple
- marcxml_values = { 'id' : '',
- 'title' : '',
- 'summary' : '',
- 'contributors' : [],
- 'journal_refs' : [],
- 'report_nos' : [],
- 'comment' : '',
- 'doi' : '' }
-
- # check if the marcxml is not empty
- if marcxml == '':
- marcxml_values['error'] = "MARCXML string is empty !"
- return marcxml_values
-
- #get the tag id and code from tag table
- main_report_number = CFG_MARC_REPORT_NUMBER
- add_report_number = CFG_MARC_ADDITIONAL_REPORT_NUMBER
- main_title = CFG_MARC_TITLE
- main_summary = CFG_MARC_ABSTRACT
- main_author = CFG_MARC_AUTHOR_NAME
- main_author_affiliation = CFG_MARC_AUTHOR_AFFILIATION
- add_author = CFG_MARC_CONTRIBUTOR_NAME
- add_author_affiliation = CFG_MARC_CONTRIBUTOR_AFFILIATION
- main_comment = CFG_MARC_COMMENT
- doi = CFG_MARC_DOI
- journal_ref_code = CFG_MARC_JOURNAL_REF_CODE
- journal_ref_title = CFG_MARC_JOURNAL_REF_TITLE
- journal_ref_page = CFG_MARC_JOURNAL_REF_PAGE
- journal_ref_year = CFG_MARC_JOURNAL_REF_YEAR
-
- #init tmp values
- contributor = {'name' : '', 'email' : '', 'affiliation' : []}
-
- try:
- bfo = BibFormatObject(recID=None, xml_record=marcxml)
- except:
- marcxml_values['error'] = "Unable to open marcxml file !"
- return marcxml_values
-
- marcxml_values = { 'id' : bfo.field(main_report_number),
- 'title' : bfo.field(main_title),
- 'summary' : bfo.field(main_summary),
- 'report_nos' : bfo.fields(add_report_number),
- 'contributors' : [],
- 'journal_refs' : [],
- 'comment' : bfo.field(main_comment),
- 'doi' : bfo.field(doi)}
-
- authors = bfo.fields(main_author[:-1], repeatable_subfields_p=True)
- for author in authors:
- name = author.get(main_author[-1], [''])[0]
- affiliation = author.get(main_author_affiliation[-1], [])
- author = {'name': name, 'email': '', 'affiliation': affiliation}
- marcxml_values['contributors'].append(author)
-
- authors = bfo.fields(add_author[:-1], repeatable_subfields_p=True)
- for author in authors:
- name = author.get(add_author[-1], [''])[0]
- affiliation = author.get(add_author_affiliation[-1], [])
- author = {'name': name, 'email': '', 'affiliation': affiliation}
- marcxml_values['contributors'].append(author)
-
- journals = bfo.fields(journal_ref_title[:-1])
- for journal in journals:
- journal_title = journal.get(journal_ref_title[-1], '')
- journal_page = journal.get(journal_ref_page[-1], '')
- journal_code = journal.get(journal_ref_code[-1], '')
- journal_year = journal.get(journal_ref_year[-1], '')
- journal = "%s: %s (%s) pp. %s" % (journal_title, journal_code, journal_year, journal_page)
- marcxml_values['journal_refs'].append(journal)
-
- return marcxml_values
-
-
-def get_subfield_value_from_datafield(datafield, field_tag):
- '''
- This function get the datafield note from a marcxml and get the tag
- value according to the tag id and code given
- @param datafield: xml node to be parsed
- @param field_tag: tuple containing id and code to find
- @return: value of the tag as a string
- '''
-
- # extract the tag number
- tag = datafield.attributes["tag"]
-
- tag_id = field_tag[0] + field_tag[1] + field_tag[2]
- tag_code = field_tag[5]
-
- # retreive the reference to the media
- if tag.value == tag_id:
- subfields = datafield.getElementsByTagName('subfield')
- for subfield in subfields:
- if subfield.attributes['code'].value == tag_code:
- return subfield.firstChild.nodeValue.encode('utf-8')
-
- return ''
-
-
-def get_list_of_marcxml_datafields(marcxml, isfile=False):
- '''
- This method parse the marcxml file to retrieve the root of the datafields
- needed by all function that format marcxml nodes.
- @param marcxml: file or string that contains the marcxml file
- @param isfile: boolean that informs if a file or a string was given
- @return: root of all datafileds
- '''
-
- #variable that contains the result of the parsing of the marcxml file
- if isfile:
- try:
- parsed_marcxml = minidom.parse(marcxml)
- except IOError:
- return 0
- else:
- parsed_marcxml = minidom.parseString(marcxml)
-
- collections = parsed_marcxml.getElementsByTagName('collection')
-
- # some macxml file has no collection root but direct record entry
- if len(collections) > 0:
- collection = collections[0]
- records = collection.getElementsByTagName('record')
- else:
- records = parsed_marcxml.getElementsByTagName('record')
-
- record = records[0]
-
- return record.getElementsByTagName('datafield')
-
-
-def format_file_to_zip_archiv(paths):
- '''
- This method takes a list of different type of file, zip its and group
- its into a zip archiv for sending
- @param paths: list of path to file of different types
- @return: (zip archiv) zipped file that contains all fulltext to submit
- '''
-
- (zip_fd, zip_path) = mkstemp(suffix='.zip', prefix='bibsword_media_',
- dir=CFG_TMPDIR)
-
- archiv = zipfile.ZipFile(zip_path, "w")
-
- for path in paths:
- if os.path.exists(path):
- archiv.write(path, os.path.basename(path), zipfile.ZIP_DEFLATED)
-
- archiv.close()
-
- return zip_path
-
-
-#-------------------------------------------------------------------------------
-# getting info from media deposit response file
-#-------------------------------------------------------------------------------
-
-def format_link_from_result(result):
- '''
- This method parses the xml file returned after the submission of a media
- and retreive the URL contained in it
- @param result: xml file returned by ArXiv
- @return: (links) table of url
- '''
- if isinstance(result, list):
- result = result[0]
-
- # parse the xml to access each node
- parsed_result = minidom.parseString(result)
-
- # finding the links in the xml file
- xml_entries = parsed_result.getElementsByTagName('entry')
- xml_entry = xml_entries[0]
- xml_contents = xml_entry.getElementsByTagName('content')
-
- # getting the unique content node
- content = xml_contents[0]
-
- # declare the dictionnary that contains type and url of a link
- link = {}
- link['link'] = content.attributes['src'].value.encode('utf-8')
- link['type'] = content.attributes['type'].value.encode('utf-8')
-
- return link
-
-
-def format_update_time_from_result(result):
- '''
- parse any xml response to retreive and format the value of the 'updated'
- tag.
- @param result: xml result of a deposit or a submit call to a server
- @return: formated date content in the node
- '''
-
- # parse the xml to access each node
- parsed_result = minidom.parseString(result)
-
- # finding the links in the xml file
- xml_entries = parsed_result.getElementsByTagName('entry')
- xml_entry = xml_entries[0]
- xml_updated = xml_entry.getElementsByTagName('updated')
-
- # getting the unique content node
- updated = xml_updated[0]
-
- return updated.firstChild.nodeValue.encode('utf-8')
-
-
-def format_links_from_submission(submission):
- '''
- parse the xml response of a metadata submission and retrieve all the
- informations proper to the link toward the media, the metadata and
- the status
- @param submission: xml response of a submission
- @return: tuple { 'medias', 'metadata', 'status' }
- '''
-
- # parse the xml to access each node
- parsed_result = minidom.parseString(submission)
-
- # finding the links in the xml file
- xml_entries = parsed_result.getElementsByTagName('entry')
- xml_entry = xml_entries[0]
- xml_links = xml_entry.getElementsByTagName('link')
-
- # getting all content nodes
- links = {'media':'', 'metadata':'', 'status':''}
-
- for link in xml_links:
-
- # declare the dictionnary that contains type and url of a link
- if link.attributes['rel'].value == 'edit-media':
- if links['media'] == '':
- links['media'] = link.attributes['href'].value.encode('utf-8')
- else:
- links['media'] = links['media'] + ', ' + \
- link.attributes['href'].value.encode('utf-8')
-
- if link.attributes['rel'].value == 'edit':
- links['metadata'] = link.attributes['href'].value.encode('utf-8')
-
- if link.attributes['rel'].value == 'alternate':
- links['status'] = link.attributes['href'].value.encode('utf-8')
-
- return links
-
-
-def format_id_from_submission(submission):
- '''
- Parse the submission file to retrieve the arxiv id retourned
- @param submission: xml file returned after the submission
- @return: string containing the arxiv id
- '''
-
- # parse the xml to access each node
- parsed_result = minidom.parseString(submission)
-
- # finding the id in the xml file
- xml_entries = parsed_result.getElementsByTagName('entry')
- xml_entry = xml_entries[0]
- xml_id = xml_entry.getElementsByTagName('id')[0]
-
- remote_id = xml_id.firstChild.nodeValue.encode('utf-8')
-
- (begin, sep, end) = remote_id.rpartition("/")
-
- remote_id = 'arXiv:'
- i = 0
- for elt in end:
- remote_id += elt
- if i == 3:
- remote_id += '.'
- i = i + 1
-
- return remote_id
-
-
-#-------------------------------------------------------------------------------
-# write information in the marc file
-#-------------------------------------------------------------------------------
-
-def update_marcxml_with_remote_id(recid, remote_id, action="append"):
- '''
- Write a new entry in the given marc file. This entry is the remote record
- id given by the server where the submission has been done
- @param remote_id: the string containing the id to add to the marc file
- return: boolean true if update done, false if problems
- '''
-
- field_tag = CFG_MARC_ADDITIONAL_REPORT_NUMBER
- tag_id = "%s%s%s" % (field_tag[0], field_tag[1], field_tag[2])
- tag_code = field_tag[5]
-
- # concatenation of the string to append to the marc file
- node = '''
- %(recid)s
-
- %(remote_id)s
-
- ''' % {
- 'recid': recid,
- 'tagid': tag_id,
- 'tagcode': tag_code,
- 'remote_id': remote_id
- }
-
- # creation of the tmp file containing the xml node to append
- (tmpfd, filename) = mkstemp(suffix='.xml', prefix='bibsword_append_remote_id_',
- dir=CFG_TMPDIR)
- tmpfile = os.fdopen(tmpfd, 'w')
- tmpfile.write(node)
- tmpfile.close()
-
- # insert a task in bibsched to add the node in the marc file
- if action == 'append':
- result = \
- task_low_level_submission('bibupload', 'BibSword', '-a', filename)
- elif action == 'delete':
- result = \
- task_low_level_submission('bibupload', 'BibSword', '-d', filename)
-
- return result
-
-
-def update_marcxml_with_info(recid, username, current_date, remote_id,
- action='append'):
- '''
- This function add a field in the marc file to informat that the
- record has been submitted to a remote server
- @param recid: id of the record to update
- '''
-
- # concatenation of the string to append to the marc file
- node = '''
- %(recid)s
-
- %(submit_info)s
-
- ''' % {
- 'recid': recid,
- 'tag': CFG_MARC_RECORD_SUBMIT_INFO,
- 'submit_info': CFG_SUBMIT_ARXIV_INFO_MESSAGE % (username, current_date, remote_id)
- }
-
- # creation of the tmp file containing the xml node to append
- (tmpfd, filename) = mkstemp(suffix='.xml', prefix='bibsword_append_submit_info_',
- dir=CFG_TMPDIR)
- tmpfile = os.fdopen(tmpfd, 'w')
- tmpfile.write(node)
- tmpfile.close()
-
- # insert a task in bibschedul to add the node in the marc file
- if action == 'append':
- result = \
- task_low_level_submission('bibupload', 'BibSword', '-a', filename)
- elif action == 'delete':
- result = \
- task_low_level_submission('bibupload', 'BibSword', '-d', filename)
-
- return result
-
-
-
-def upload_fulltext(recid, path):
- '''
- This method save the uploaded file to associated record
- @param recid: id of the record
- @param path: uploaded document to store
- '''
-
- # upload the file to the record
-
- bibarchiv = BibRecDocs(recid)
- docname = path.split('/')[-1].split('.')[0]
- doctype = path.split('.')[-1].split(';')[0]
- bibarchiv.add_new_file(path, CFG_DOCTYPE_UPLOAD_COLLECTION, docname,
- format=doctype)
-
- return ''
-
-
-#-------------------------------------------------------------------------------
-# work with the remote submission status xml file
-#-------------------------------------------------------------------------------
-
-def format_submission_status(status_xml):
- '''
- This method parse the given atom xml status string and retrieve the
- the value of the tag
- @param status_xml: xml atom entry
- @return: dictionnary containing status, id and/or possible error
- '''
-
- result = {'status':'', 'id_submission':'', 'error':''}
-
- parsed_status = minidom.parseString(status_xml)
- deposit = parsed_status.getElementsByTagName('deposit')[0]
- status_node = deposit.getElementsByTagName('status')[0]
- if status_node.firstChild != None:
- status = status_node.firstChild.nodeValue.encode('utf-8')
- else:
- result['status'] = ''
- return result
-
- #status = "submitted"
- if status == CFG_SUBMISSION_STATUS_SUBMITTED:
- result['status'] = status
- return result
-
- #status = "published"
- if status == CFG_SUBMISSION_STATUS_PUBLISHED:
- result['status'] = status
- arxiv_id_node = deposit.getElementsByTagName('arxiv_id')[0]
- result['id_submission'] = \
- arxiv_id_node.firstChild.nodeValue.encode('utf-8')
- return result
-
- #status = "onhold"
- if status == CFG_SUBMISSION_STATUS_ONHOLD:
- result['status'] = status
- return result
-
- #status = "removed"
- if status == 'unknown':
- result['status'] = CFG_SUBMISSION_STATUS_REMOVED
- error_node = deposit.getElementsByTagName('error')[0]
- result['error'] = error_node.firstChild.nodeValue.encode('utf-8')
- return result
-
- return result
-
-
-#-------------------------------------------------------------------------------
-# Classes for the generation of XML Atom entry containing submission metadata
-#-------------------------------------------------------------------------------
-
-class BibSwordFormat:
- '''
- This class gives the methodes needed to format all mandatories xml atom
- entry nodes. It is extended by subclasses that has optional nodes add
- to the standard SWORD format
- '''
-
- def __init__(self):
- ''' No init necessary for this class '''
-
- def frmt_id(self, recid):
- '''
- This methode check if there is an id for the resource. If it is the case,
- it format it returns a formated id node that may be inserted in the
- xml metadata file
- @param recid: the id of the resource
- @return: (xml) xml node correctly formated
- '''
-
- if recid != '':
- return '''%s \n''' % recid
- return ''
-
-
- def frmt_title(self, title):
- '''
- This methode check if there is a title for the resource. If yes,
- it returns a formated title node that may be inserted in the
- xml metadata file
- @param title: the title of the resource
- @return: (xml) xml node correctly formated
- '''
-
- if title != '':
- return '''%s \n''' % title
- return ''
-
-
- def frmt_author(self, author_name, author_email):
- '''
- This methode check if there is a submitter for the resource. If yes,
- it returns a formated author node that may containing the name and
- the email of the author to be inserted in the xml metadata file
- @param author_name: the name of the submitter of the resource
- @param author_email: the email where the remote server send answers
- @return: (xml) xml node correctly formated
- '''
-
- author = ''
- if author_name != '':
- author += '''\n'''
- author += '''%s \n''' % author_name
- if author_email != '':
- author += '''%s \n''' % author_email
- author += ''' \n'''
- return author
-
-
- def frmt_summary(self, summary):
- '''
- This methode check if there is a summary for the resource. If yes,
- it returns a formated summary node that may be inserted in the
- xml metadata file
- @param summary: the summary of the resource
- @return: (xml) xml node correctly formated
- '''
-
- if summary != '':
- return '''%s \n''' % summary
- return ''
-
-
- def frmt_categories(self, categories, scheme):
- '''
- This method check if there is some categories for the resource. If it
- is the case, it returns the categorie nodes formated to be insered in
- the xml metadata file
- @param categories: list of categories for one resource
- @return: (xml) xml node(s) correctly formated
- '''
-
- output = ''
-
- for category in categories:
-
- output += '''\n'''
-
- #id
- if 'id' in metadata:
- output += BibSwordFormat.frmt_id(self, metadata['id'])
-
- #title
- if 'title' in metadata:
- output += BibSwordFormat.frmt_title(self,
- metadata['title'])
-
- #author
- if 'author_name' in metadata and 'author_email' in metadata:
- output += BibSwordFormat.frmt_author(self, metadata['author_name'],
- metadata['author_email'])
-
- #contributors
- if 'contributors' in metadata:
- output += '' + self.frmt_contributors(metadata['contributors'])
-
- #summary
- if 'summary' in metadata:
- output += BibSwordFormat.frmt_summary(self, metadata['summary'])
-
- #categories
- if 'categories' in metadata:
- output += BibSwordFormat.frmt_categories(self, metadata['categories'],
- 'http://arxiv.org/terms/arXiv/')
-
- #primary_category
- if 'primary_url' in metadata and 'primary_label' in metadata:
- output += self.frmt_primary_category(metadata['primary_url'],
- metadata['primary_label'],
- 'http://arxiv.org/terms/arXiv/')
-
- #comment
- if 'comment' in metadata:
- output += self.frmt_comment(metadata['comment'])
-
- #journal references
- if 'journal_refs' in metadata:
- output += self.frmt_journal_ref(metadata['journal_refs'])
-
- #report numbers
- if 'report_nos' in metadata:
- output += self.frmt_report_no(metadata['report_nos'])
-
- #doi
- if 'doi' in metadata:
- output += self.frmt_doi(metadata['doi'])
-
- #link
- if 'links' in metadata:
- output += BibSwordFormat.frmt_link(self, metadata['links'])
-
- output += ''' '''
-
- return output
-
-
- def frmt_contributors(self, contributors):
- '''
- This method display each contributors in the format of an editable input
- text. This allows the user to modifie it.
- @param contributors: The list of all contributors of the document
- @return: (html code) the html code that display each dropdown list
- '''
-
- output = ''
-
- for contributor in contributors:
- output += '''\n'''
- output += '''%s \n''' % contributor['name']
- if contributor['email'] != '':
- output += '''%s \n''' % \
- contributor['email']
- if len(contributor['affiliation']) != 0:
- for affiliation in contributor['affiliation']:
- output += '''%s'''\
- ''' \n''' % affiliation
- output += ''' \n'''
-
- return output
-
-
- def frmt_primary_category(self, primary_url, primary_label, scheme):
- '''
- This method format the primary category as an element of a dropdown
- list.
- @param primary_url: url of the primary category deposit
- @param primary_label: name of the primary category to display
- @param scheme: url of the primary category schema
- @return: html code containing each element to display
- '''
-
- output = ''
-
- if primary_url != '':
- output += '''%s \n''' % comment
-
- return output
-
-
- def frmt_journal_ref(self, journal_refs):
- '''
- This method check if there is some journal refs for the resource. If it
- is the case, it returns the journal_ref nodes formated to be insered in
- the xml metadata file
- @param journal_refs: list of journal_refs for one resource
- @return: (xml) xml node(s) correctly formated
- '''
-
- output = ''
-
- for journal_ref in journal_refs:
- output += '''%s \n''' % \
- journal_ref
-
- return output
-
-
- def frmt_report_no(self, report_nos):
- '''
- This method check if there is some report numbres for the resource. If it
- is the case, it returns the report_nos nodes formated to be insered in
- the xml metadata file
- @param report_nos: list of report_nos for one resource
- @return: (xml) xml node(s) correctly formated
- '''
-
- output = ''
-
- for report_no in report_nos:
- output += '''%s \n''' % \
- report_no
-
- return output
-
-
- def frmt_doi(self, doi):
- '''This methode check if there is an doi given. If it is the case, it
- format it returns a formated doi node that may be inserted in the xml
- metadata file
- @param doi: the string doi
- @return: (xml) xml node correctly formated
- '''
-
- output = ''
-
- if doi != '':
- output = '''%s \n''' % doi
-
- return output
diff --git a/modules/bibsword/lib/bibsword_client_http.py b/modules/bibsword/lib/bibsword_client_http.py
deleted file mode 100644
index 96563c0ec0..0000000000
--- a/modules/bibsword/lib/bibsword_client_http.py
+++ /dev/null
@@ -1,188 +0,0 @@
-# This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
-#
-# Invenio is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-#
-# Invenio is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Invenio; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
-'''
-BibSWORD Client Http Queries
-'''
-
-import urllib2
-from tempfile import NamedTemporaryFile
-from invenio.config import CFG_TMPDIR
-from invenio.urlutils import make_user_agent_string
-
-class RemoteSwordServer:
- '''This class gives every tools to communicate with the SWORD/APP deposit
- of ArXiv.
- '''
-
- # static variable used to properly perform http request
- agent = make_user_agent_string("BibSWORD")
-
-
- def __init__(self, authentication_infos):
-
- '''
- This method the constructor of the class, it initialise the
- connection using a passord. That allows users to connect with
- auto-authentication.
- @param self: reference to the current instance of the class
- @param authentication_infos: dictionary with authentication infos containing
- keys:
- - realm: realm of the server
- - hostname: hostname of the server
- - username: name of an arxiv known user
- - password: password of the known user
- '''
-
- #password manager with default realm to avoid looking for it
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-
- passman.add_password(authentication_infos['realm'],
- authentication_infos['hostname'],
- authentication_infos['username'],
- authentication_infos['password'])
-
- #create an authentificaiton handler
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
-
- http_handler = urllib2.HTTPHandler(debuglevel=0)
-
- opener = urllib2.build_opener(authhandler, http_handler)
- # insalling : every call to opener will user the same user/pass
- urllib2.install_opener(opener)
-
-
- def get_remote_collection(self, url):
- '''
- This method sent a request to the servicedocument to know the
- collections offer by arxives.
- @param self: reference to the current instance of the class
- @param url: the url where the request is made
- @return: (xml file) collection of arxiv allowed for the user
- '''
-
- #format the request
- request = urllib2.Request(url)
-
- #launch request
- #try:
- response = urllib2.urlopen(request)
- #except urllib2.HTTPError:
- # return ''
- #except urllib2.URLError:
- # return ''
-
- return response.read()
-
-
- def deposit_media(self, media, collection, onbehalf):
- '''
- This method allow the deposit of any type of media on a given arxiv
- collection.
- @param self: reference to the current instanc off the class
- @param media: dict of file info {'type', 'size', 'file'}
- @param collection: abreviation of the collection where to deposit
- @param onbehalf: user that make the deposition
- @return: (xml file) contains error ot the url of the temp file
- '''
-
- #format the final deposit URL
- deposit_url = collection
-
- #prepare the header
- headers = {}
- headers['Content-Type'] = media['type']
- headers['Content-Length'] = media['size']
- #if on behalf, add to the header
- if onbehalf != '':
- headers['X-On-Behalf-Of'] = onbehalf
-
- headers['X-No-Op'] = 'True'
- headers['X-Verbose'] = 'True'
- headers['User-Agent'] = self.agent
-
- #format the request
- result = urllib2.Request(deposit_url, media['file'], headers)
-
- #launch request
- try:
- return urllib2.urlopen(result).read()
- except urllib2.HTTPError:
- return ''
-
-
- def metadata_submission(self, deposit_url, metadata, onbehalf):
- '''
- This method send the metadata to ArXiv, then return the answere
- @param metadata: xml file to submit to ArXiv
- @param onbehalf: specify the persone (and email) to informe of the
- publication
- '''
-
- #prepare the header of the request
- headers = {}
- headers['Host'] = 'arxiv.org'
- headers['User-Agent'] = self.agent
- headers['Content-Type'] = 'application/atom+xml;type=entry'
- #if on behalf, add to the header
- if onbehalf != '':
- headers['X-On-Behalf-Of'] = onbehalf
-
- headers['X-No-Op'] = 'True'
- headers['X-verbose'] = 'True'
-
- #format the request
- result = urllib2.Request(deposit_url, metadata, headers)
-
- #launch request
- try:
- response = urllib2.urlopen(result).read()
- except urllib2.HTTPError, e:
- tmpfd = NamedTemporaryFile(mode='w', suffix='.xml', prefix='bibsword_error_',
- dir=CFG_TMPDIR, delete=False)
- tmpfd.write(e.read())
- tmpfd.close()
- return ''
- except urllib2.URLError:
- return ''
-
- return response
-
-
- def get_submission_status(self, status_url) :
- '''
- This method get the xml file from the given URL and return it
- @param status_url: url where to get the status
- @return: xml atom entry containing the status
- '''
-
- #format the http request
- request = urllib2.Request(status_url)
- request.add_header('Host', 'arxiv.org')
- request.add_header('User-Agent', self.agent)
-
- #launch request
- try:
- response = urllib2.urlopen(request).read()
- except urllib2.HTTPError:
- return 'HTTPError (Might be an authentication issue)'
- except urllib2.URLError:
- return 'Wrong url'
-
- return response
-
-
diff --git a/modules/bibsword/lib/bibsword_client_server.py b/modules/bibsword/lib/bibsword_client_server.py
new file mode 100644
index 0000000000..faef8ff092
--- /dev/null
+++ b/modules/bibsword/lib/bibsword_client_server.py
@@ -0,0 +1,521 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+Base class for the SWORD client servers.
+"""
+
+import urllib2
+from socket import getdefaulttimeout, setdefaulttimeout
+from mimetypes import guess_all_extensions
+from tempfile import NamedTemporaryFile
+from datetime import datetime, timedelta
+import re
+
+from invenio.config import CFG_TMPDIR
+from invenio.dbquery import(
+ run_sql,
+ serialize_via_marshal,
+ deserialize_via_marshal
+)
+from invenio.dateutils import(
+ convert_datestruct_to_datetext
+)
+
+from invenio.bibsword_config import \
+ CFG_BIBSWORD_USER_AGENT, \
+ CFG_BIBSWORD_FILE_TYPES_MAPPING, \
+ CFG_BIBSWORD_DEFAULT_TIMEOUT, \
+ CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT, \
+ CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE
+
+
+class SwordClientServer(object):
+ """
+ Base class for the SWORD client servers.
+ Missing funcionality needs to be extended by specific implementations.
+ """
+
+ def __init__(self, settings):
+ """
+ Initialize the object by setting the instance variables,
+ decompressing the parsed service document and
+ preparing the auth handler.
+ """
+
+ # We expect to have the following instance variables:
+ # self.server_id (int)
+ # self.name (str)
+ # self.username (str)
+ # self.password (str)
+ # self.email (str)
+ # self.service_document_parsed (blob)
+ # self.update_frequency (str)
+ # self.last_updated (str/timestamp)
+ # self.realm (str)
+ # self.uri (str)
+ # self.service_document_url (str)
+
+ for (name, value) in settings.iteritems():
+ setattr(self, name, value)
+
+ self._prepare_auth_handler()
+
+ if self.service_document_parsed:
+ self.service_document_parsed = deserialize_via_marshal(
+ self.service_document_parsed
+ )
+ else:
+ self.update()
+
+ def _prepare_auth_handler(self):
+ """
+ """
+
+ # create a password manager
+ self._password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+
+ # Add the username and password
+ self._password_mgr.add_password(self.realm,
+ self.uri,
+ self.username,
+ self.password)
+
+ def _url_opener(self):
+ """
+ """
+
+ # Create a basic authentication handler
+ _auth_handler = urllib2.HTTPBasicAuthHandler(self._password_mgr)
+
+ # Return a URL opener (OpenerDirector instance)
+ return urllib2.build_opener(_auth_handler)
+
+ @staticmethod
+ def _get_extension_for_file_type(file_type):
+
+ extensions = guess_all_extensions(file_type)
+
+ if not extensions:
+ def _guess_extension_for_ambiguous_file_type(ambiguous_file_type):
+ return ambiguous_file_type.split('/')[-1]
+ extensions = CFG_BIBSWORD_FILE_TYPES_MAPPING.get(
+ file_type,
+ _guess_extension_for_ambiguous_file_type(file_type))
+
+ return extensions
+
+ def submit(self, metadata, media, url):
+ """
+ """
+
+ # Perform the media deposit
+ media_response = self._deposit_media(
+ metadata,
+ media,
+ url
+ )
+
+ # Perform the metadata submission
+ metadata_response = self._ingest_metadata(
+ metadata,
+ media_response,
+ url
+ )
+
+ # Prepare and return the final response
+ response = self._prepare_response(
+ media_response,
+ metadata_response
+ )
+
+ return response
+
+ def _deposit_media(self, metadata, media, url):
+ """
+ """
+
+ # Hold the response for each file in the response dictionary
+ response = {}
+
+ prepared_media = self._prepare_media(media)
+
+ for (file_index, file_info) in prepared_media.iteritems():
+ headers = self._prepare_media_headers(file_info, metadata)
+ try:
+ file_object = open(file_info['path'], "r")
+ file_contents = file_object.read()
+ file_object.close()
+ except Exception, e:
+ response[file_index] = {
+ 'error': True,
+ 'msg': str(e),
+ 'name': file_info['name'],
+ 'mime': file_info['mime'],
+ }
+ else:
+ req = urllib2.Request(url, file_contents, headers)
+ try:
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ default_timeout = getdefaulttimeout()
+ setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT)
+ res = self._url_opener().open(req)
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ setdefaulttimeout(default_timeout)
+ except urllib2.HTTPError, e:
+ response[file_index] = {
+ 'error': True,
+ 'msg': self._prepare_media_response_error(e),
+ 'name': file_info['name'],
+ 'mime': file_info['mime'],
+ }
+ except Exception, e:
+ response[file_index] = {
+ 'error': True,
+ 'msg': str(e),
+ 'name': file_info['name'],
+ 'mime': file_info['mime'],
+ }
+ else:
+ response[file_index] = {
+ 'error': False,
+ 'msg': self._prepare_media_response(res),
+ 'name': file_info['name'],
+ 'mime': file_info['mime'],
+ }
+
+ return response
+
+ def _prepare_media(self, media):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return media
+
+ def _prepare_media_headers(self, file_info, dummy):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ headers = {}
+
+ headers['Content-Type'] = file_info['mime']
+ headers['Content-Length'] = file_info['size']
+ headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT
+
+ return headers
+
+ def _prepare_media_response(self, response):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return response.read()
+
+ def _prepare_media_response_error(self, error):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return error
+
+ def _ingest_metadata(self, metadata, media_response, url):
+ """
+ """
+
+ response = {}
+
+ if media_response:
+ for file_response in media_response.itervalues():
+ if file_response['error']:
+ response['error'] = True
+ response['msg'] = (
+ "There has been at least one error with " +
+ "the files chosen for media deposit."
+ )
+ return response
+ else:
+ response['error'] = True
+ response['msg'] = (
+ "No files were chosen for media deposit. " +
+ "At least one file has to be chosen."
+ )
+ return response
+
+ prepared_metadata = self._prepare_metadata(metadata, media_response)
+
+ headers = self._prepare_metadata_headers(metadata)
+
+ req = urllib2.Request(url, prepared_metadata, headers)
+ try:
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ default_timeout = getdefaulttimeout()
+ setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT)
+ res = self._url_opener().open(req)
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ setdefaulttimeout(default_timeout)
+ except urllib2.HTTPError, e:
+ response['error'] = True
+ response['msg'] = self._prepare_metadata_response_error(e)
+ except Exception, e:
+ response['error'] = True
+ response['msg'] = str(e)
+ else:
+ response['error'] = False
+ response['msg'] = self._prepare_metadata_response(res)
+
+ return response
+
+ def _prepare_metadata(self, metadata, dummy):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return metadata
+
+ def _prepare_metadata_headers(self, dummy):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ headers = {}
+
+ headers['Content-Type'] = CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE
+ headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT
+
+ return headers
+
+ def _prepare_metadata_response(self, response):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return response.read()
+
+ def _prepare_metadata_response_error(self, error):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return error
+
+ def _prepare_response(self, media_response, metadata_response):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return None
+
+ def status(self, url):
+ """
+ """
+
+ response = {}
+
+ headers = self._prepare_status_headers()
+
+ req = urllib2.Request(url, headers=headers)
+
+ try:
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ default_timeout = getdefaulttimeout()
+ setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT)
+ # TODO: No need for authentication at this point,
+ # maybe use the standard url_opener?
+ res = self._url_opener().open(req)
+ if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT:
+ setdefaulttimeout(default_timeout)
+ except urllib2.HTTPError, e:
+ response['error'] = True
+ response['msg'] = str(e)
+ except Exception, e:
+ response['error'] = True
+ response['msg'] = str(e)
+ else:
+ response['error'] = False
+ response['msg'] = self._prepare_status_response(res)
+ self._store_submission_status(
+ url,
+ response['msg']['error'] is None and "{0}".format(
+ response['msg']['status']
+ ) or "{0} ({1})".format(
+ response['msg']['status'],
+ response['msg']['error']
+ )
+ )
+
+ return response
+
+ def _prepare_status_headers(self):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ headers = {}
+
+ headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT
+
+ return headers
+
+ def _prepare_status_response(self, response):
+ """
+ """
+
+ # NOTE: Implement me for each server!
+
+ return response.read()
+
+ def _store_submission_status(self, url, status):
+ """
+ Stores the submission status in the database.
+ """
+
+ query = """
+ UPDATE swrCLIENTSUBMISSION
+ SET status=%s,
+ last_updated=%s
+ WHERE id_server=%s
+ AND url_alternate=%s
+ """
+
+ params = (
+ status,
+ convert_datestruct_to_datetext(datetime.now()),
+ self.server_id,
+ url,
+ )
+
+ result = run_sql(query, params)
+
+ return result
+
+ @staticmethod
+ def _convert_frequency_to_timedelta(raw_frequency):
+ """
+ Converts this: "5w4d3h2m1s"
+ to this: datetime.timedelta(39, 10921)
+ """
+
+ frequency_translation = {
+ "w": "weeks",
+ "d": "days",
+ "h": "hours",
+ "m": "minutes",
+ "s": "seconds",
+ }
+
+ frequency_parts = re.findall(
+ "([0-9]+)([wdhms]{1})",
+ raw_frequency
+ )
+
+ frequency_parts = map(
+ lambda p: (frequency_translation[p[1]], int(p[0])),
+ frequency_parts
+ )
+
+ frequency_timedelta_parts = dict(frequency_parts)
+
+ frequency_timedelta = timedelta(**frequency_timedelta_parts)
+
+ return frequency_timedelta
+
+ def needs_to_be_updated(self):
+ """
+ Check if the service document is out of date,
+ i.e. if the "last_updated" date is more than
+ "update_frequency" time in the past.
+ """
+
+ frequency_timedelta = \
+ SwordClientServer._convert_frequency_to_timedelta(
+ self.update_frequency
+ )
+
+ if ((self.last_updated + frequency_timedelta) < datetime.now()):
+ return True
+
+ return False
+
+ def update(self):
+ """
+ Fetches the latest service document, parses it and
+ saves it in the database.
+ """
+
+ service_document_raw = self._fetch_service_document()
+
+ if service_document_raw:
+ service_document_parsed = self._parse_service_document(
+ service_document_raw
+ )
+
+ if service_document_parsed:
+ self.service_document_parsed = service_document_parsed
+ return self._store_service_document()
+
+ return False
+
+ def _fetch_service_document(self):
+ """
+ Returns the raw service document of the server.
+ """
+
+ req = urllib2.Request(self.service_document_url)
+ try:
+ res = self._url_opener().open(req)
+ except urllib2.HTTPError:
+ return None
+ service_document_raw = res.read()
+ res.close()
+
+ return service_document_raw
+
+ def _store_service_document(self):
+ """
+ Stores the compressed parsed service document in the database.
+ """
+
+ query = """
+ UPDATE swrCLIENTSERVER
+ SET service_document_parsed=%s,
+ last_updated=%s
+ WHERE id=%s
+ """
+
+ params = (
+ serialize_via_marshal(self.service_document_parsed),
+ convert_datestruct_to_datetext(datetime.now()),
+ self.server_id
+ )
+
+ result = run_sql(query, params)
+
+ return result
diff --git a/modules/bibsword/lib/bibsword_client_templates.py b/modules/bibsword/lib/bibsword_client_templates.py
index b347dc927a..cc35ed9703 100644
--- a/modules/bibsword/lib/bibsword_client_templates.py
+++ b/modules/bibsword/lib/bibsword_client_templates.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
-
+#
# This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -17,1090 +17,2029 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-'''
-BibSWORD Client Templates
-'''
-
-from invenio.config import CFG_SITE_URL, CFG_SITE_NAME, CFG_SITE_RECORD
-
-class BibSwordTemplate:
- '''
- This class contains attributes and methods that allows to display all
- information used by the BibSword web user interface. Theses informations
- are form, validation or error messages
- '''
-
- def __init__(self):
- ''' No init necessary for this class '''
-
- #---------------------------------------------------------------------------
- # BibSword WebSubmit Interface
- #---------------------------------------------------------------------------
-
- def tmpl_display_submit_ack(self, remote_id, link):
- '''
- This method generate the html code that displays the acknoledgement
- message after the submission of a record.
- @param remote_id: id of the record given by arXiv
- @param link: links to modify or consult submission
- @return: string containing the html code
- '''
-
- html = ''
-
- html += '''Success ! '''
- html += '''The record has been successfully pushed to arXiv !
'''
- html += '''The arXiv id of the submission is: %s
''' % \
- remote_id
- html += '''Manage your submission
'''
+"""
+BibSWORD Client Templates.
+"""
+
+from cgi import escape
+from datetime import datetime
+import re
+from invenio.config import(
+ CFG_SITE_RECORD,
+ CFG_SITE_URL
+)
+from invenio.messages import gettext_set_language
+
+
+class TemplateSwordClient(object):
+ """
+ Templates for the Sword client.
+ """
+
+ @staticmethod
+ def _tmpl_submission_table_row(
+ submission,
+ ln
+ ):
+ """
+ Returns the HTML code for a server table row.
+ """
+ (user,
+ record_id,
+ server_id,
+ server_name,
+ submitted,
+ status,
+ last_updated,
+ status_url
+ ) = submission
+
+ table_row = """
+
+ """.format(record_id, server_id)
+
+ table_row += """
+
+ {0}
+
+ """.format(user)
+
+ table_row += """
+
+ #{2}
+
+ """.format(CFG_SITE_URL, CFG_SITE_RECORD, record_id)
+
+ table_row += """
+
+ {0}
+
+ """.format(server_name)
+
+ table_row += """
+
+ {0}
+
+ """.format(
+ TemplateSwordClient._humanize_datetime(
+ submitted,
+ ln
+ )
+ )
+
+ table_row += """
+
+ {0}
+
+ """.format(status)
+
+ table_row += """
+
+ {0}
+
+ """.format(
+ TemplateSwordClient._humanize_datetime(
+ last_updated,
+ ln
+ )
+ )
+
+ # ↺ --> ↺
+ html_options = """
+ ↺
+ """
+
+ table_row += """
+
+ {0}
+
+ """.format(html_options.format(server_id, status_url, ln))
+
+ table_row += """
+
+ """
+
+ return table_row
+
+ def tmpl_submissions(
+ self,
+ submissions,
+ ln
+ ):
+ """
+ Returns the submissions table with all information and options.
+ """
+
+ _ = gettext_set_language(ln)
+
+ html = """
+
+
+
+
+
+
+ {user_header_label}
+
+
+ {record_header_label}
+
+
+ {server_header_label}
+
+
+ {submitted_header_label}
+
+
+ {status_header_label}
+
+
+ {updated_header_label}
+
+
+ {options_header_label}
+
+
+
+
+ {tbody}
+
+
+
+
+ """
+
+ html_tbody = ""
+
+ for submission in submissions:
+ html_tbody += TemplateSwordClient._tmpl_submission_table_row(
+ submission,
+ ln
+ )
+
+ return html.format(
+ error_message=_("An error has occured. " +
+ "The administrators have been informed."),
+ user_header_label=_("User"),
+ record_header_label=_("Record"),
+ server_header_label=_("Server"),
+ submitted_header_label=_("Submitted"),
+ status_header_label=_("Status"),
+ updated_header_label=_("Last updated"),
+ options_header_label=_("Options"),
+ tbody=html_tbody
+ )
+
+ @staticmethod
+ def _tmpl_server_table_row(
+ server,
+ ln
+ ):
+ """
+ Returns the HTML code for a server table row.
+ """
+ (server_id,
+ name,
+ engine,
+ username,
+ dummy,
+ email,
+ dummy,
+ update_frequency,
+ last_updated) = server
+
+ table_row = """
+
+ """.format(server_id)
+
+ table_row += """
+
+ {0}
+
+ """.format(name)
+
+ table_row += """
+
+ {0}
+
+ """.format(engine)
+
+ table_row += """
+
+ {0}
+
+ """.format(username)
+
+ table_row += """
+
+ {0}
+
+ """.format(email)
+
+ table_row += """
+
+ {0}
+
+ """.format(
+ TemplateSwordClient._humanize_frequency(
+ update_frequency,
+ ln
+ )
+ )
+
+ table_row += """
+
+ {0}
+
+ """.format(
+ TemplateSwordClient._humanize_datetime(
+ last_updated,
+ ln
+ )
+ )
+
+ # ↺ --> ↺
+ # ✎ --> ✎
+ # ✗ --> ✗
+ html_options = """
+ ↺
+
+ ✎
+
+ ✗
+ """
+
+ table_row += """
+
+ {0}
+
+ """.format(html_options.format(server_id, ln))
+
+ table_row += """
+
+ """
+
+ return table_row
+
+ def tmpl_servers(
+ self,
+ servers,
+ ln
+ ):
+ """
+ Returns the servers table with all information and available options.
+ """
+
+ _ = gettext_set_language(ln)
+
+ # ⊕ --> ⊕
+ html = """
+
+
+
+
+
+
+
+
+ {name_header_label}
+
+
+ {engine_header_label}
+
+
+ {username_header_label}
+
+
+ {email_header_label}
+
+
+ {frequency_header_label}
+
+
+ {last_updated_header_label}
+
+
+ {options_header_label}
+
+
+
+
+ {tbody}
+
+
+
+
+ """
+
+ html_tbody = ""
+
+ for server in servers:
+ html_tbody += TemplateSwordClient._tmpl_server_table_row(
+ server,
+ ln
+ )
+
+ return html.format(
+ confirm_delete_label=_(
+ "Are you sure you want to delete this server?"
+ ),
+ add_label=_("Add server"),
+ error_message=_("An error has occured. " +
+ "The administrators have been informed."),
+ name_header_label=_("Name"),
+ engine_header_label=_("Engine"),
+ username_header_label=_("Username"),
+ email_header_label=_("E-mail"),
+ frequency_header_label=_("Update frequency"),
+ last_updated_header_label=_("Last updated"),
+ options_header_label=_("Options"),
+ tbody=html_tbody,
+ ln=ln
+ )
+
+ def tmpl_add_or_modify_server(
+ self,
+ server,
+ available_engines,
+ ln
+ ):
+ """
+ Returns the servers table with all information and available options.
+ """
+
+ _ = gettext_set_language(ln)
+
+ if server:
+ (server_id,
+ name,
+ engine,
+ username,
+ password,
+ email,
+ dummy,
+ update_frequency,
+ dummy) = server
+ label = _("Modify server")
+ content = """
+
+
+
+ {title}:
+
+
+
+ {subtitle}
+
+
+
+
+
{label}
+ """
+
+ html_select = """
+
+
+
+ {title}:
+
+
+
+ {subtitle}
+
+
+
+
+ {options}
+
+
+
{label}
+ """.format(
+ label=_("Cancel")
+ )
+
+ controls += """
+
+ """
+
+ controls += """
+ {label}
+ """.format(
+ label=_("Submit")
+ )
+
+ html = """
+
+
+
+
+
+ {label}
+
+
+
+
+ {controls}
+
+
+
\n''' \
- ''' \n''' \
- ''' ID \n''' \
- ''' %(server_id)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Name \n''' \
- ''' %(server_name)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Host \n''' \
- ''' %(server_host)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Username \n''' \
- ''' %(username)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Password \n''' \
- ''' %(password)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Email \n''' \
- ''' %(email)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Realm \n''' \
- ''' %(realm)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' Record URL \n''' \
- ''' %(url_base_record)s \n''' \
- ''' \n ''' \
- ''' \n''' \
- ''' URL Servicedocument \n'''\
- ''' %(url_servicedocument)s \n''' \
- ''' \n ''' \
- '''
''' % {
- 'table_width' : '50%',
- 'server_id' : server_info['server_id'],
- 'server_name' : server_info['server_name'],
- 'server_host' : server_info['server_host'],
- 'username' : server_info['username'],
- 'password' : server_info['password'],
- 'email' : server_info['email'],
- 'realm' : server_info['realm'],
- 'url_base_record' : server_info['url_base_record'],
- 'url_servicedocument': server_info['url_servicedocument']
- }
-
- return body
-
-
- def tmpl_display_remote_servers(self, remote_servers, id_record,
- error_messages):
- '''
- format the html code that display a dropdown list containing the
- servers
- @param self: reference to the current instance of the class
- @param remote_servers: list of tuple containing server's infos
- @return: string containing html code
- '''
-
- body = '''
-''' % {
- 'error_message': \
- self.display_error_message_row(error_messages),
- 'table_width' : '100%',
- 'row_width' : '50%',
- 'id_record' : id_record,
- 'remote_server': \
- self.fill_dropdown_remote_servers(remote_servers)
- }
-
- return body
-
-
- def tmpl_display_collections(self, selected_server, server_infos,
- collections, id_record, recid, error_messages):
- '''
- format the html code that display the selected server, the informations
- about the selected server and a dropdown list containing the server's
- collections
- @param self: reference to the current instance of the class
- @param selected_server: tuple containing selected server name and id
- @param server_infos: tuple containing infos about selected server
- @param collections: list contianing server's collections
- @return: string containing html code
- '''
-
- body = '''
-''' % {
- 'table_width' : '100%',
- 'row_width' : '50%',
- 'error_message' : \
- self.display_error_message_row(error_messages),
- 'id_server' : selected_server['id'],
- 'server_name' : selected_server['name'],
- 'server_version' : server_infos['version'],
- 'server_maxUpload': server_infos['maxUploadSize'],
- 'collection' : \
- self.fill_dropdown_collections(collections),
- 'id_record' : id_record,
- 'recid' : recid
- }
-
- return body
-
-
- def tmpl_display_categories(self, selected_server, server_infos,
- selected_collection, collection_infos,
- primary_categories, secondary_categories,
- id_record, recid, error_messages):
- '''
- format the html code that display the selected server, the informations
- about the selected server, the selected collections, the informations
- about the collection and a dropdown list containing the server's
- primary and secondary categories
- @param self: reference to the current instance of the class
- @param selected_server: tuple containing selected server name and id
- @param server_infos: tuple containing infos about selected server
- @param selected_collection: selected collection
- @param collection_infos: tuple containing infos about selected col
- @param primary_categories: list of mandated categories for the col
- @return: string containing html code
- '''
-
- body = '''
-''' % {
- 'table_width' : '100%',
- 'row_width' : '50%',
- 'error_message' : self.display_error_message_row(
- error_messages),
-
- # hidden input
- 'id_server' : selected_server['id'],
- 'id_collection' : selected_collection['id'],
- 'id_record' : id_record,
- 'recid' : recid,
-
- # variables values
- 'server_name' : selected_server['name'],
- 'server_version' : server_infos['version'],
- 'server_maxUpload' : server_infos['maxUploadSize'],
-
- 'collection_name' : selected_collection['label'],
-
- 'collection_accept': ''.join([
- '''%(name)s ''' % {
- 'name': accept
- } for accept in collection_infos['accept'] ]),
-
- 'collection_url' : selected_collection['url'],
- 'primary_categories' : self.fill_dropdown_primary(
- primary_categories),
-
- 'secondary_categories': self.fill_dropdown_secondary(
- secondary_categories)
- }
-
- return body
-
-
- def tmpl_display_metadata(self, user, server, collection, primary,
- categories, medias, metadata, id_record, recid,
- error_messages):
- '''
- format a string containing every informations before a submission
- '''
-
-
- body = '''
-''' % {
- 'table_width' : '100%',
- 'submissions' : self.fill_submission_table(submissions),
- 'CFG_SITE_URL' : CFG_SITE_URL
- }
-
- return body
-
-
- #***************************************************************************
- # Private functions
- #***************************************************************************
-
-
- def display_error_message_row(self, error_messages):
- '''
- return a list of error_message in form of a bullet list
- @param error_messages: list of error_messages to display
- @return: html code that display list of errors
- '''
-
- # if no errors, return nothing
- if len(error_messages) == 0:
- return ''
-
- if len(error_messages) == 1:
- # display a generic header message
- body = '''
-
-
-
- The following error was found:
-
-'''
+ @staticmethod
+ def _humanize_frequency(
+ raw_frequency,
+ ln
+ ):
+ """
+ Converts this: "3w4d5h6m7s"
+ to this: "Every 3 weeks, 4 days, 5 hours, 6 minutes, 7 seconds"
+ """
+
+ _ = gettext_set_language(ln)
+
+ frequency_translation = {
+ "w": _("week(s)"),
+ "d": _("day(s)"),
+ "h": _("hour(s)"),
+ "m": _("minute(s)"),
+ "s": _("second(s)"),
+ }
+
+ humanized_frequency = _("Every")
+ humanized_frequency += " "
+
+ frequency_parts = re.findall(
+ "([0-9]+)([wdhms]{1})",
+ raw_frequency
+ )
+
+ frequency_parts = map(
+ lambda p: "{0} {1}".format(p[0], frequency_translation[p[1]]),
+ frequency_parts
+ )
+
+ humanized_frequency += ", ".join(frequency_parts)
+
+ return humanized_frequency
+
+ @staticmethod
+ def _humanize_datetime(
+ raw_datetime,
+ ln
+ ):
+ """
+ Converts this: "2015-02-17 10:10:10"
+ to this: "2 week(s), 5 day(s) ago"
+ """
+
+ _ = gettext_set_language(ln)
+
+ if not raw_datetime:
+ return _("Never")
+
+ passed_datetime = datetime.now() - raw_datetime
+
+ passed_datetime_days = passed_datetime.days
+
+ humanized_datetime_parts = []
+
+ if passed_datetime_days:
+ passed_datetime_weeks = passed_datetime_days / 7
+ if passed_datetime_weeks:
+ humanized_datetime_parts.append(
+ "{0!s} {1}".format(passed_datetime_weeks, _("week(s)"))
+ )
+ passed_datetime_days = passed_datetime_days % 7
+ humanized_datetime_parts.append(
+ "{0!s} {1}".format(passed_datetime_days, _("day(s)"))
+ )
else:
- # display a generic header message
- body = '''
-
-
-
- Following errors were found:
-
-'''
-
- # insert each error lines
- for error_message in error_messages:
- body = body + '''
- %(error)s ''' % {
- 'error': error_message
+ passed_datetime_seconds = passed_datetime.seconds
+ passed_datetime_minutes = passed_datetime_seconds / 60
+ passed_datetime_hours = passed_datetime_minutes / 60
+ if passed_datetime_hours:
+ humanized_datetime_parts.append(
+ "{0!s} {1}".format(passed_datetime_hours, _("hour(s)"))
+ )
+ passed_datetime_minutes = passed_datetime_minutes % 60
+ if passed_datetime_minutes:
+ humanized_datetime_parts.append(
+ "{0!s} {1}".format(passed_datetime_minutes, _("minute(s)"))
+ )
+ passed_datetime_seconds = passed_datetime_seconds % 60
+ if passed_datetime_seconds:
+ humanized_datetime_parts.append(
+ "{0!s} {1}".format(passed_datetime_seconds, _("second(s)"))
+ )
+
+ humanized_datetime = ", ".join(humanized_datetime_parts)
+ humanized_datetime += " "
+ humanized_datetime += _("ago")
+
+ return humanized_datetime
+
+ @staticmethod
+ def _submit_prepare_select_tag(
+ select_id,
+ select_name,
+ options,
+ default_option=None,
+ multiple=False
+ ):
+ """Prepare the HTML select tag."""
+ out = """
+
+ """ % (
+ select_id,
+ select_name and ' name="%s"' % (select_name,) or '',
+ multiple and ' multiple="multiple"' or ''
+ )
+
+ for option in options:
+ option_value = option[0]
+ option_text = option[1]
+ out += """
+ %s
+ """ % (
+ escape(str(option_value), True),
+ default_option is not None and (
+ option_value == default_option and " selected" or ""
+ ) or "",
+ escape(str(option_text), True)
+ )
+
+ out += """
+
+ """
+
+ return out
+
+ def tmpl_submit(
+ self,
+ sid,
+ recid,
+ title,
+ author,
+ report_number,
+ step,
+ servers,
+ server_id,
+ ln
+ ):
+ """
+ The main submission interface.
+
+ Through a series of steps, the user is guided through the submission.
+ """
+
+ _ = gettext_set_language(ln)
+
+ out = """
+
+
+
+
+
+ %(step_1_title)s
+
+
+
+ %(step_1_details)s
+
+
+
+ %(step_2_title)s
+
+
+
+ %(step_2_details)s
+
+
+
+ %(step_3_title)s
+
+
+
+ %(step_3_details)s
+
+
+
+ %(step_4_title)s
+
+
+
+ %(step_4_details)s
+
+
+
+ %(step_final_title)s
+
+
+
+ %(step_final_details)s
+
+
+
{title} {author}
+ """.format(
+ submitting=_("You are submitting:"),
+ CFG_SITE_URL=CFG_SITE_URL,
+ CFG_SITE_RECORD=CFG_SITE_RECORD,
+ record_id=record_id,
+ title=escape(title, True),
+ report_number=escape(report_number, True),
+ by=_("by"),
+ author=escape(author, True)
+ )
+
+ return out
+
+ def _submit_loading(self, ln):
+ """
+ NOTE_2015
+ """
+
+ _ = gettext_set_language(ln)
+
+ out = """
+ %s
+ """ % (
+ "bibsword_submit_step_1_confirm",
+ _("Confirm and continue")
+ )
+
+ out = """
+
+
+ %(label)s
+
+
+ %(selection)s
+
+
%s
+ """ % (
+ "bibsword_submit_step_2_confirm",
+ _("Confirm and continue")
+ )
+
+ out = """
+
+
+ %(label)s
+
+
+ %(selection)s
+
+
%s
+ """ % (
+ "bibsword_submit_step_3_collection_confirm",
+ _("Confirm and continue")
+ )
+
+ # TODO: Different SWORD servers will have different concepts
+ # of categories (mandatory, optional, etc). Write a function
+ # that accomodates all cases.
+ if not mandatory_categories:
+ mandatory_categories = optional_categories
+ optional_categories = None
+
+ out = """
+
+
+ %(label)s
+
+
+ %(primary_category_selection)s
+
+
%s
+ """ % (
+ "bibsword_submit_step_3_optional_categories_add",
+ "" + _("✚") + " "
+ )
+
+ out += """
+
+
+ %(label)s
+
+
+ %(additional_category_selection)s
+ %(addition)s
+
+
+
+
+ %(sortable_label)s
+ %(remove_label)s
+
+
+ %(sortable_label)s
+
+
+
+
+
+ %(remove_label)s
+
+
+ %(label)s
+
+ """ % {
+ 'label': _('Contributors:'),
+ }
- # if it is the first media of this type, create a new type
- is_type_in_media_type = False
- for type in media_type:
- if media['collection'] == type['media_type']:
- is_type_in_media_type = True
+ if len(metadata['contributors']) > maximum_number_of_contributors:
+ contributors_text += """
+
+ %(text)s
+
+ """ % {
+ 'text': _('Displaying only the first {0}').format(maximum_number_of_contributors),
+ }
+
+ contributors_text += """
+
+ """
+
+ for contributor in metadata['contributors'][:maximum_number_of_contributors]:
+
+ contributors_text += TemplateSwordClient._submit_step_4_contributors_input_block(
+ fullname=(
+ _('Fullname:'),
+ 'contributor_fullname',
+ contributor[0] and escape(contributor[0], True) or '',
+ '25',
+ _('Fullname...'),
+ ),
+ email=(
+ _('E-mail:'),
+ 'contributor_email',
+ contributor[1] and escape(contributor[1], True) or '',
+ '35',
+ _('E-mail...'),
+ ),
+ affiliation=(
+ _('Affiliation:'),
+ 'contributor_affiliation',
+ contributor[2] and escape(contributor[2], True) or '',
+ '35',
+ _('Affiliation...'),
+ ),
+ sortable_label=_("⇳"),
+ remove_class='bibsword_submit_step_4_contributors_remove negative',
+ remove_label=_("✘")
+ )
+
+ contributors_text += """
+
+ """
+
+ contributors_text += """
+
+
+ %(insert_label)s %(label)s
+
+
+ """ % {
+ 'insert_class': 'bibsword_submit_step_4_contributors_insert positive',
+ 'insert_label': _("✚"),
+ 'label': _("Insert another one..."),
+ }
- if is_type_in_media_type == False:
- type = {}
- type['media_type'] = media['collection']
- type['media_list'] = []
- media_type.append(type)
+ contributors_text += """
+
+
+ %(label)s
+
+ """ % {
+ 'label': _('Additional report numbers:'),
+ }
- # insert the media in the good media_type element
- for type in media_type:
- if type['media_type'] == media['collection']:
- type['media_list'].append(media)
+ additional_rn_text += """
+
+ """
+
+ for additional_rn in metadata['additional_rn']:
+
+ additional_rn_text += TemplateSwordClient._submit_step_4_additional_rn_input_block(
+ name='additional_rn',
+ value=additional_rn and escape(additional_rn, True) or '',
+ size='25',
+ placeholder=_('Report number...'),
+ sortable_label=_("⇳"),
+ remove_class='bibsword_submit_step_4_additional_rn_remove negative',
+ remove_label=_("✘")
+ )
+
+ additional_rn_text += """
+
+ """
+
+ additional_rn_text += """
+
+
+ %(insert_label)s %(label)s
+
+
+ """ % {
+ 'insert_class': 'bibsword_submit_step_4_additional_rn_insert positive',
+ 'insert_label': _("✚"),
+ 'label': _("Insert another one..."),
+ }
+
+ additional_rn_text += """
+
+
+ %(label)s
+
+ """ % {
+ 'label': _('Files'),
+ }
+
+ for file_key in files.iterkeys():
+
+ files_text += """
+
+ """ % {
+ 'name': 'files',
+ 'value': str(file_key),
+ 'url': escape(files[file_key]['url'], True),
+ 'label': escape(files[file_key]['name'], True),
+ }
+
+ files_text += """
+
+ """ % {
+ "id": "bibsword_submit_step_4_submission_data",
+ }
- return media_type
+ text_body = (
+ rn_text +
+ additional_rn_text +
+ title_text +
+ author_text +
+ abstract_text +
+ contributors_text +
+ files_text +
+ confirmation_text
+ )
+
+ text_close = """
+
+ """
+
+ return text_open + text_body + text_close
+
+ def submit_step_final_details(
+ self,
+ ln
+ ):
+ """
+ Return the HTML for the final step details.
+ """
+
+ _ = gettext_set_language(ln)
+
+ out = _("The submission has been completed successfully.here .").format(ln)
+
+ return out
diff --git a/modules/bibsword/lib/bibsword_client_tester.py b/modules/bibsword/lib/bibsword_client_tester.py
deleted file mode 100644
index 778d6fe5ef..0000000000
--- a/modules/bibsword/lib/bibsword_client_tester.py
+++ /dev/null
@@ -1,668 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
-#
-# Invenio is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version.
-#
-# Invenio is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Invenio; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
-"""Test cases for the BibSword module."""
-
-__revision__ = "$Id$"
-
-# pylint: disable-msg=C0301
-
-from invenio.testutils import InvenioTestCase
-import os
-import sys
-import time
-from invenio.testutils import make_test_suite, run_test_suite
-from invenio.config import CFG_TMPDIR
-from invenio.bibsword_client_formatter import format_marcxml_file, \
- format_submission_status, \
- ArXivFormat
-from invenio.bibsword_client import format_metadata, \
- list_submitted_resources, \
- perform_submission_process
-
-from invenio.bibsword_client_http import RemoteSwordServer
-from invenio.bibsword_client_dblayer import get_remote_server_auth, \
- insert_into_swr_clientdata, \
- update_submission_status, \
- select_submitted_record_infos, \
- delete_from_swr_clientdata
-from invenio.dbquery import run_sql
-from invenio.bibsword_config import CFG_SUBMISSION_STATUS_SUBMITTED, \
- CFG_SUBMISSION_STATUS_PUBLISHED, \
- CFG_SUBMISSION_STATUS_REMOVED
-from xml.dom import minidom
-
-class Test_format_marcxml_file(InvenioTestCase):
- """ bibsword_format - test the parsing and extracting of marcxml nodes"""
-
- def test_extract_marcxml_1(self):
- """Test_format_marcxml_file - extract marcxml without id, report_nos and comment"""
- # Test with marcxml file 1
- marcxml = open("%s%sTest_marcxml_file_1.xml" % (CFG_TMPDIR, os.sep)).read()
- metadata = format_marcxml_file(marcxml)
- self.assertEqual(metadata['id'], '')
- self.assertEqual(metadata['title'], "Calorimetry triggering in ATLAS")
- self.assertEqual(metadata['contributors'][0]['name'], "Igonkina, O")
- self.assertEqual(metadata['contributors'][0]['affiliation'][0], "NIKHEF, Amsterdam")
- self.assertEqual(metadata['summary'], "The ATLAS experiment is preparing for data taking at 14 TeV collision energy. A rich discovery physics program is being prepared in addition to the detailed study of Standard Model processes which will be produced in abundance. The ATLAS multi-level trigger system is designed to accept one event in 2 105 to enable the selection of rare and unusual physics events. The ATLAS calorimeter system is a precise instrument, which includes liquid Argon electro-magnetic and hadronic components as well as a scintillator-tile hadronic calorimeter. All these components are used in the various levels of the trigger system. A wide physics coverage is ensured by inclusively selecting events with candidate electrons, photons, taus, jets or those with large missing transverse energy. The commissioning of the trigger system is being performed with cosmic ray events and by replaying simulated Monte Carlo events through the trigger and data acquisition system.")
- self.assertEqual(metadata['contributors'][1]['name'], "Achenbach, R")
- self.assertEqual(metadata['contributors'][1]['affiliation'][0], "Kirchhoff Inst. Phys.")
- self.assertEqual(metadata['contributors'][2]['name'], "Adragna, P")
- self.assertEqual(metadata['contributors'][2]['affiliation'][0], "Queen Mary, U. of London")
- nb_contributors = len(metadata['contributors'])
- self.assertEqual(nb_contributors, 205)
- self.assertEqual(metadata['contributors'][204]['name'], "Özcan, E")
- self.assertEqual(metadata['contributors'][204]['affiliation'][0], "University Coll. London")
- self.assertEqual(metadata['doi'], "10.1088/1742-6596/160/1/012061")
- self.assertEqual(metadata['journal_refs'][0], "J. Phys.: Conf. Ser.: 012061 (2009) pp. 160")
- self.assertEqual(len(metadata['journal_refs']), 1)
- self.assertEqual('report_nos' in metadata, True)
- self.assertEqual(metadata['comment'], '')
-
-
- def test_extract_marcxml_2(self):
- """Test_format_marcxml_file - extract marcxml without report_nos, doi"""
-
- #Test with marcxml file 2
- marcxml = open("%s%sTest_marcxml_file_2.xml" % (CFG_TMPDIR, os.sep)).read()
- metadata = format_marcxml_file(marcxml)
- self.assertEqual(metadata['id'], "arXiv:1001.1674")
- self.assertEqual(metadata['title'], "Persistent storage of non-event data in the CMS databases")
- self.assertEqual(metadata['contributors'][0]['name'], "De Gruttola, M")
- self.assertEqual(metadata['contributors'][0]['affiliation'][0], "CERN")
- self.assertEqual(metadata['contributors'][0]['affiliation'][1], "INFN, Naples")
- self.assertEqual(metadata['contributors'][0]['affiliation'][2], "Naples U.")
- self.assertEqual(metadata['summary'], "In the CMS experiment, the non event data needed to set up the detector, or being produced by it, and needed to calibrate the physical responses of the detector itself are stored in ORACLE databases. The large amount of data to be stored, the number of clients involved and the performance requirements make the database system an essential service for the experiment to run. This note describes the CMS condition database architecture, the data-flow and PopCon, the tool built in order to populate the offline databases. Finally, the first results obtained during the 2008 and 2009 cosmic data taking are presented.")
- self.assertEqual(metadata['contributors'][1]['name'], "Di Guida, S")
- self.assertEqual(metadata['contributors'][1]['affiliation'][0], "CERN")
- self.assertEqual(metadata['contributors'][2]['name'], "Futyan, D")
- self.assertEqual(metadata['contributors'][2]['affiliation'][0], "Imperial Coll., London")
- nb_contributors = len(metadata['contributors'])
- self.assertEqual(nb_contributors, 11)
- self.assertEqual(metadata['contributors'][10]['name'], "Xie, Z")
- self.assertEqual(metadata['contributors'][10]['affiliation'][0], "Princeton U.")
- self.assertEqual(metadata['doi'], '')
- self.assertEqual(metadata['journal_refs'][0], "JINST: P04003 (2010) pp. 5")
- self.assertEqual(len(metadata['journal_refs']), 1)
- self.assertEqual('report_nos' in metadata, True)
- self.assertEqual(metadata['comment'], "Comments: 20 pages, submitted to IOP")
-
- def test_extract_full_marcxml_3(self):
- """Test_format_marcxml_file - extract marcxml without doi"""
-
- #Test with marcxml file 3
- marcxml = open("%s%sTest_marcxml_file_3.xml" % (CFG_TMPDIR, os.sep)).read()
- metadata = format_marcxml_file(marcxml)
- self.assertEqual(metadata['id'], "ATL-PHYS-CONF-2007-008")
- self.assertEqual(metadata['title'], "Early Standard Model physics and early discovery strategy in ATLAS")
- self.assertEqual(metadata['contributors'][0]['name'], "Grosse-Knetter, J")
- self.assertEqual(metadata['contributors'][0]['affiliation'][0], "Bonn U.")
- self.assertEqual(metadata['summary'], "In 2008 the LHC will open a new energy domain for physics within the Standard Model and beyond. The physics channels which will be addressed by the ATLAS experiment in the initial period of operation will be discussed. These include Standard Model processes such as W/Z production and early top measurements. This will be followed by a description of the searches for a low-mass Higgs boson, new heavy di-lepton resonances, and Supersymmetry, for which a striking signal might be observed after only a few months of data taking.")
- self.assertEqual(len(metadata['contributors']), 1)
- self.assertEqual(metadata['doi'], '')
- self.assertEqual(metadata['journal_refs'][0], "Nucl. Phys. B, Proc. Suppl.: 55-59 (2008) pp. 177-178")
- self.assertEqual(len(metadata['journal_refs']), 1)
- self.assertEqual(len(metadata['report_nos']), 2)
- self.assertEqual(metadata['report_nos'][0], "ATL-COM-PHYS-2007-036")
- self.assertEqual(metadata['report_nos'][1], "CERN-ATL-COM-PHYS-2007-036")
- self.assertEqual(metadata['comment'], '')
-
- def test_extract_null_marcxml(self):
- """Test_format_marcxml_file - no metadata"""
-
- #Test without any marcxml file
- metadata = format_marcxml_file("")
- self.assertEqual(metadata["error"], "MARCXML string is empty !")
-
- def test_extract_wrong_file(self):
- """Test_format_marcxml_file - unexistant metadata file"""
-
- #Test with a unknown marcxml file
- metadata = format_marcxml_file("%s%sTest_marcxml_file_false.xml" % (CFG_TMPDIR, os.sep), True)
- self.assertEqual(metadata["error"], "Unable to open marcxml file !")
-
-
-class Test_format_metadata(InvenioTestCase):
- """ bibsword - test the collection of all metadata """
-
- def test_correct_metadata_collection(self):
- """Test_format_metadata - collection of metadata without errors"""
-
- marcxml = open("%s%sTest_marcxml_file_3.xml" % (CFG_TMPDIR, os.sep)).read()
-
- metadata = {}
- metadata['primary_label'] = 'Test - Test Disruptive Networks'
- metadata['primary_url'] = 'http://arxiv.org/terms/arXiv/test.dis-nn'
-
- user_info = {}
- user_info['nickname'] = 'test_user'
- user_info['email'] = 'test@user.com'
-
- deposit_results = []
- deposit_results.append(open("%s%sTest_media_deposit.xml" % (CFG_TMPDIR, os.sep)).read())
- metadata = format_metadata(marcxml, deposit_results, user_info, metadata)
-
- self.assertEqual(metadata['id'], "ATL-PHYS-CONF-2007-008")
- self.assertEqual(metadata['title'], "Early Standard Model physics and early discovery strategy in ATLAS")
- self.assertEqual(metadata['contributors'][0]['name'], "Grosse-Knetter, J")
- self.assertEqual(metadata['contributors'][0]['affiliation'][0], "Bonn U.")
- self.assertEqual(metadata['summary'], "In 2008 the LHC will open a new energy domain for physics within the Standard Model and beyond. The physics channels which will be addressed by the ATLAS experiment in the initial period of operation will be discussed. These include Standard Model processes such as W/Z production and early top measurements. This will be followed by a description of the searches for a low-mass Higgs boson, new heavy di-lepton resonances, and Supersymmetry, for which a striking signal might be observed after only a few months of data taking.")
- nb_contributors = len(metadata['contributors'])
- self.assertEqual(nb_contributors, 1)
- self.assertEqual(metadata['doi'], "")
- self.assertEqual(len(metadata['journal_refs']), 1)
- self.assertEqual(metadata['journal_refs'][0], "Nucl. Phys. B, Proc. Suppl.: 55-59 (2008) pp. 177-178")
- self.assertEqual(len(metadata['report_nos']), 2)
- self.assertEqual(metadata['report_nos'][0], "ATL-COM-PHYS-2007-036")
- self.assertEqual(metadata['report_nos'][1], "CERN-ATL-COM-PHYS-2007-036")
- self.assertEqual(metadata['comment'], "")
-
- self.assertEqual(metadata['primary_label'], "Test - Test Disruptive Networks")
- self.assertEqual(metadata['primary_url'], "http://arxiv.org/terms/arXiv/test.dis-nn")
-
- self.assertEqual(metadata['author_name'], "test_user")
- self.assertEqual(metadata['author_email'], "test@user.com")
-
- self.assertEqual(metadata['links']['link'], "https://arxiv.org/sword-app/edit/10070072")
- self.assertEqual(metadata['links']['type'], "application/pdf")
-
-
- def test_metadata_collection_no_data(self):
- """Test_format_metadata - collection of metadata without any changes"""
-
- # Gives an empty marcxml file
- marcxml = ""
- # Gives no metadata
- metadata = {}
- # Gives no user informations
- user_info = {}
- # Gives no result where to find a link
- deposit_results = []
-
- metadata = format_metadata(marcxml, deposit_results, user_info, metadata)
- self.assertEquals(len(metadata['error']), 9)
- self.assertEquals(metadata['error'][0], "No submitter name given !")
- self.assertEquals(metadata['error'][1], "No submitter email given !")
- self.assertEquals(metadata['error'][2], "No primary category label given !")
- self.assertEquals(metadata['error'][3], "No primary category url given !")
- self.assertEquals(metadata['error'][4], "No links to the media deposit found !")
- self.assertEquals(metadata['error'][5], "No document id given !")
- self.assertEquals(metadata['error'][6], "No title given !")
- self.assertEquals(metadata['error'][7], "No author given !")
- self.assertEquals(metadata['error'][8], "No summary given !")
-
-
- self.assertEquals(metadata['id'], "")
- self.assertEquals(metadata['title'], "")
- self.assertEqual(len(metadata['contributors']), 0)
- self.assertEqual(metadata['summary'], "")
- self.assertEqual(metadata['doi'], "")
- self.assertEqual(len(metadata['journal_refs']), 0)
- self.assertEqual(len(metadata['report_nos']), 0)
- self.assertEqual(metadata['comment'], "")
- self.assertEqual(metadata['primary_label'], "")
- self.assertEqual(metadata['primary_url'], "")
- self.assertEqual(metadata['author_name'], "")
- self.assertEqual(metadata['author_email'], "")
- self.assertEqual(len(metadata['links']), 0)
-
-
-class Test_get_submission_status(InvenioTestCase):
- """ bibsword_httpquery - test the get_submission_status method """
-
- def test_get_submission_status_ok(self):
- """Test_get_submission_status - connect to an existing url"""
-
- authentication_info = get_remote_server_auth(4)
- connection = RemoteSwordServer(authentication_info)
- result = connection.get_submission_status("http://arxiv.org/resolve/app/10070073")
-
- self.assertEqual(result != "", True)
-
-
- def test_get_submission_status_no_url(self):
- """Test_get_submission_status - connect to an existing url"""
-
- authentication_info = get_remote_server_auth(4)
- connection = RemoteSwordServer(authentication_info)
- result = connection.get_submission_status("")
-
- self.assertEqual(result != "", False)
-
- def test_get_submission_status_wrong_url(self):
- """Test_get_submission_status - connect to an existing url"""
-
- authentication_info = get_remote_server_auth(4)
- connection = RemoteSwordServer(authentication_info)
- result = connection.get_submission_status("http://arxiv.org/reso")
-
- self.assertEqual(result != "", False)
-
-
-class Test_format_submission_status(InvenioTestCase):
- """bibsword_format - test the parsing of format_submission_status method"""
-
- def test_format_submission_status_submitted(self):
- """Test_format_submission_status_submitted"""
-
- status_xml = open("%s%sTest_submission_status_submitted.xml" % (CFG_TMPDIR, os.sep)).read()
- response = format_submission_status(status_xml)
-
- self.assertEqual(response['status'], "submitted")
- self.assertEqual(response['id_submission'], "")
- self.assertEqual(response['error'], "")
-
-
- def test_format_submission_status_published(self):
- """Test_format_submission_status_published"""
-
- status_xml = open("%s%sTest_submission_status_published.xml" % (CFG_TMPDIR, os.sep)).read()
- response = format_submission_status(status_xml)
-
- self.assertEqual(response['status'], "published")
- self.assertEqual(response['id_submission'], "1003.9876")
- self.assertEqual(response['error'], "")
-
-
- def test_format_submission_status_onhold(self):
- """Test_format_submission_status_onhold"""
-
- status_xml = open("%s%sTest_submission_status_onhold.xml" % (CFG_TMPDIR, os.sep)).read()
- response = format_submission_status(status_xml)
-
- self.assertEqual(response['status'], "onhold")
- self.assertEqual(response['id_submission'], "")
- self.assertEqual(response['error'], "")
-
-
- def test_format_submission_status_removed(self):
- """Test_format_submission_status_removed"""
-
- status_xml = open("%s%sTest_submission_status_unknown.xml" % (CFG_TMPDIR, os.sep)).read()
- response = format_submission_status(status_xml)
-
- self.assertEqual(response['status'], CFG_SUBMISSION_STATUS_REMOVED)
- self.assertEqual(response['id_submission'], "")
- self.assertEqual(response['error'], "identifier does not correspond to a SWORD wrapper, it may belong to a media deposit")
-
-
-class Test_swrCLIENTDATA_table(InvenioTestCase):
- '''
- This test check that the entire update process works fine. It insert some
- data into the swrCLIENTDATA table, then he get some xml status entry from
- ArXiv and Finally, it update those that have changed their status
- '''
-
- id_tests = []
-
- def test_insert_submission(self):
- '''Test_insert_submission - check insert submission rows in swrCLIENTDATA'''
-
- self.id_tests.append(insert_into_swr_clientdata(1, 97, 'TESLA-FEL-99-07', 10030148,
- '6', 'test_username', 'juliet.capulet@cds.cern.ch',
- 'test_media_deposit', 'test_media_submit',
- 'https://arxiv.org/sword-app/edit/10030148',
- 'https://arxiv.org/sword-app/edit/10030148.atom',
- 'http://arxiv.org/resolve/app/10030148'))
-
- self.id_tests.append(insert_into_swr_clientdata(1, 92, 'hep-th/0606096', 10070221,
- '5', 'test_username', 'romeo.montague@cds.cern.ch',
- 'test_media_deposit', 'test_media_submit',
- 'https://arxiv.org/sword-app/edit/10070221',
- 'https://arxiv.org/sword-app/edit/10070221.atom',
- 'http://arxiv.org/resolve/app/10070221'))
-
- self.id_tests.append(insert_into_swr_clientdata(1, 92, 'hep-th/0606096', 12340097,
- '5', 'test_username', 'romeo.montague@cds.cern.ch',
- 'test_media_deposit', 'test_media_submit',
- 'https://arxiv.org/sword-app/edit/12340097',
- 'https://arxiv.org/sword-app/edit/12340097.atom',
- 'http://arxiv.org/resolve/app/12340097'))
-
- rows = run_sql('''SELECT id, id_swrREMOTESERVER, id_record, report_no,
- id_remote, id_user, user_name, user_email, xml_media_deposit,
- xml_metadata_submit, submission_date, publication_date, removal_date,
- link_medias, link_metadata, link_status, status, last_update
- FROM swrCLIENTDATA''')
-
- for row in rows:
- self.assertEqual(row[0] in self.id_tests, True)
- if row[0] == self.id_tests[0]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 97)
- self.assertEqual(row[4], '10030148')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], "https://arxiv.org/sword-app/edit/10030148")
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10030148.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10030148')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED)
-
- if row[0] == self.id_tests[1]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 92)
- self.assertEqual(row[4], '10070221')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10070221')
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10070221.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10070221')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED)
-
- if row[0] == self.id_tests[2]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 92)
- self.assertEqual(row[4], '12340097')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/12340097')
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/12340097.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/12340097')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED)
-
-
- def test_update_submission(self):
- '''Test_insert_submission - check update submission rows in swrCLIENTDATA'''
-
- update_submission_status(self.id_tests[0], CFG_SUBMISSION_STATUS_SUBMITTED)
- update_submission_status(self.id_tests[1], CFG_SUBMISSION_STATUS_PUBLISHED, '1007.0221')
- update_submission_status(self.id_tests[2], CFG_SUBMISSION_STATUS_REMOVED)
-
- rows = run_sql('''SELECT id, id_swrREMOTESERVER, id_record, report_no,
- id_remote, id_user, user_name, user_email, xml_media_deposit,
- xml_metadata_submit, submission_date, publication_date, removal_date,
- link_medias, link_metadata, link_status, status, last_update
- FROM swrCLIENTDATA''')
- for row in rows:
- self.assertEqual(row[0] in self.id_tests, True)
- if row[0] == self.id_tests[0]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 97)
- self.assertEqual(row[4], '10030148')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10030148')
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10030148.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10030148')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED)
-
- if row[0] == self.id_tests[1]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 92)
- self.assertEqual(row[4], '1007.0221')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10070221')
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10070221.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10070221')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_PUBLISHED)
-
- if row[0] == self.id_tests[2]:
- self.assertEqual(row[1], 1)
- self.assertEqual(row[2], 92)
- self.assertEqual(row[4], '12340097')
- self.assertEqual(row[8], 'test_media_deposit')
- self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/12340097')
- self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/12340097.atom')
- self.assertEqual(row[15], 'http://arxiv.org/resolve/app/12340097')
- self.assertEqual(row[16], CFG_SUBMISSION_STATUS_REMOVED)
-
-
- def test_yread_submission(self):
- '''test_read_submission - check read submission rows in swrCLIENTDATA'''
-
-
- currentDate = time.strftime("%Y-%m-%d %H:%M:%S")
-
- results = select_submitted_record_infos()
-
- self.assertEqual(len(results), run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0])
- for result in results:
- self.assertEqual(result['id'] in self.id_tests, True)
- if result['id'] == self.id_tests[0]:
- self.assertEqual(result['id_server'], 1)
- self.assertEqual(result['id_record'], 97)
- self.assertEqual(result['id_user'], 6)
- self.assertEqual(result['id_remote'], '10030148')
- self.assertEqual(result['submission_date'], currentDate)
- self.assertEqual(result['publication_date'], '')
- self.assertEqual(result['removal_date'], "")
- self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/10030148')
- self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/10030148.atom')
- self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/10030148')
- self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_SUBMITTED)
-
- results = select_submitted_record_infos(4)
-
- self.assertEqual(len(results), run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA WHERE id_swrREMOTESERVER=4''')[0][0])
-
- for result in results:
- self.assertEqual(result['id'] in self.id_tests, True)
- if result['id'] == self.id_tests[1]:
- self.assertEqual(result['id_server'], 4)
- self.assertEqual(result['id_record'], 92)
- self.assertEqual(result['id_user'], 5)
- self.assertEqual(result['id_remote'], '1007.0221')
- self.assertEqual(result['submission_date'], currentDate)
- self.assertEqual(result['publication_date'], currentDate)
- self.assertEqual(result['removal_date'], "")
- self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/10070221')
- self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/10070221.atom')
- self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/10070221')
- self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_PUBLISHED)
-
-
- results = select_submitted_record_infos(row_id=self.id_tests[2])
- self.assertEqual(len(results), 1)
-
- for result in results:
- self.assertEqual(result['id'] in self.id_tests, True)
- if result['id'] == self.id_tests[2]:
- self.assertEqual(result['id_server'], 1)
- self.assertEqual(result['id_record'], 92)
- self.assertEqual(result['id_user'], 5)
- self.assertEqual(result['id_remote'], '12340097')
- self.assertEqual(result['submission_date'], currentDate)
- self.assertEqual(result['publication_date'], "")
- self.assertEqual(result['removal_date'], currentDate)
- self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/12340097')
- self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/12340097.atom')
- self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/12340097')
- self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_REMOVED)
-
-
- def test_zdelete_submission(self):
- '''test_delete_submission - check delete submission rows in swrCLIENTDATA'''
-
- nb_rows_before = run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0]
-
- for id_test in self.id_tests:
- delete_from_swr_clientdata(id_test)
-
- nb_rows_after = run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0]
-
- nb_rows = nb_rows_before - nb_rows_after
-
- self.assertEqual(nb_rows, 3)
-
-
-class Test_list_submitted_resources(InvenioTestCase):
- '''Test_list_submitted_resources - check the data selection and update for admin interface'''
-
- def test_list_submitted_resources(self):
- '''Test_list_submitted_resources - check the data selection and update for admin interface'''
-
- id_tests = []
-
- id_tests.append(insert_into_swr_clientdata(4, 97, 1, 'test', 'test', '10030148',
- 'https://arxiv.org/sword-app/edit/10030148',
- 'https://arxiv.org/sword-app/edit/10030148.atom',
- 'http://arxiv.org/resolve/app/10030148'))
-
- time.sleep(1)
-
- id_tests.append(insert_into_swr_clientdata(4, 97, 1, 'test', 'test', '10030148',
- 'https://arxiv.org/sword-app/edit/10030148',
- 'https://arxiv.org/sword-app/edit/10030148.atom',
- 'http://arxiv.org/resolve/app/10030148'))
-
- update_submission_status(id_tests[1], CFG_SUBMISSION_STATUS_PUBLISHED, '1003.0148')
-
- time.sleep(1)
-
- id_tests.append(insert_into_swr_clientdata(3, 92, 2, 'test', 'test', '12340097',
- 'https://arxiv.org/sword-app/edit/12340097',
- 'https://arxiv.org/sword-app/edit/12340097.atom',
- 'http://arxiv.org/resolve/app/12340097'))
-
- time.sleep(1)
-
- (submissions, modifications) = list_submitted_resources(0, 10, '')
-
- self.assertEqual(len(submissions), 3)
- self.assertEqual(len(modifications), 2)
-
- self.assertEqual(submissions[1]['id_remote'], '1003.3743')
- self.assertEqual(submissions[1]['status'], CFG_SUBMISSION_STATUS_PUBLISHED)
- self.assertEqual(submissions[1]['publication_date'] != '', True)
-
- self.assertEqual(submissions[2]['id_remote'], '1003.0148')
- self.assertEqual(submissions[2]['status'], CFG_SUBMISSION_STATUS_PUBLISHED)
- self.assertEqual(submissions[2]['publication_date'] != '', True)
-
- self.assertEqual(submissions[0]['id_remote'], '12340097')
- self.assertEqual(submissions[0]['status'], CFG_SUBMISSION_STATUS_REMOVED)
- self.assertEqual(submissions[0]['removal_date'] != '', True)
-
- self.assertEqual(modifications[1], submissions[0]['id'])
- self.assertEqual(modifications[0], submissions[1]['id'])
-
-
- for id_test in id_tests:
- delete_from_swr_clientdata(id_test)
-
-
-class Test_format_metadata_atom(InvenioTestCase):
- ''' Test_format_metadata_atom - check the generation of the atom entry containing metadata'''
-
- def test_format_full_metadata(self):
- ''' test_format_full_metadata check that every metadata get its xml node'''
-
- contributors = []
- contributor = {'name' : 'contributor_1_test',
- 'email' : 'contributor_1@test.com',
- 'affiliation': 'affiliation_test'}
- contributors.append(contributor)
-
- contributor = {'name' : 'contributor_2_test',
- 'email' : 'contributor_2@test.com',
- 'affiliation': ''}
- contributors.append(contributor)
-
- contributor = {'name' : 'contributor_3_test',
- 'email' : '',
- 'affiliation' : ''}
- contributors.append(contributor)
-
- categories = []
- category = {'url' : 'https://arxiv.org/test-categories-1',
- 'label': 'category_1_test'}
- categories.append(category)
- category = {'url' : 'https://arxiv.org/test-categories-2',
- 'label': 'category_2_test'}
- categories.append(category)
- category = {'url' : 'https://arxiv.org/test-categories-3',
- 'label': 'category_3_test'}
- categories.append(category)
-
- journal_refs = []
- journal_refs.append('journal_ref_test_1')
- journal_refs.append('journal_ref_test_2')
-
- report_nos = []
- report_nos.append('report_no_test_1')
- report_nos.append('report_no_test_2')
-
- links = []
- links.append({'link': 'http://arxiv.org/test_1',
- 'type': 'application/test_1'})
- links.append({'link': 'http://arxiv.org/test_2',
- 'type': 'application/test_2'})
-
- metadata = {'id' : 'id_test',
- 'title' : 'title_test',
- 'author_name' : 'author_test',
- 'author_email': 'author_email',
- 'contributors': contributors,
- 'summary' : 'summary_test',
- 'categories' : categories,
- 'primary_url' : 'https://arxiv.org/primary-test-categories',
- 'primary_label': 'primary_category_label_test',
- 'comment' : '23 pages, 3 chapters 1 test',
- 'journal_refs': journal_refs,
- 'report_nos' : report_nos,
- 'doi' : '10.2349.test.209',
- 'links' : links
- }
-
- arXivFormat = ArXivFormat()
- metadata_atom = arXivFormat.format_metadata(metadata)
-
- entry_node = minidom.parseString(metadata_atom)
-
-
- self.assertEqual(metadata_atom != '', True)
-
-
-class Test_submission_process(InvenioTestCase):
- '''Test_submission_process - test document submission'''
-
- def test_perform_submission_process(self):
- '''Test_perform_submission_process - test document submission'''
-
- metadata = {}
- metadata['primary_label'] = 'Test - Test Disruptive Networks'
- metadata['primary_url'] = 'http://arxiv.org/terms/arXiv/test.dis-nn'
-
- user_info = {}
- user_info['nickname'] = 'test_user'
- user_info['email'] = 'test@user.com'
- user_info['id'] = 1
-
- result = perform_submission_process(4, 'https://arxiv.org/sword-app/test-collection',
- 97, user_info, metadata)
-
- self.assertEqual(open('/tmp/media.xml', 'r').read() != '', True)
- self.assertEqual(open('/tmp/metadata.xml', 'r').read() != '', True)
- self.assertEqual(open('/tmp/submit.xml', 'r').read() != '', True)
-
-
-
- if result['row_id'] != '':
- delete_from_swr_clientdata(result['row_id'])
-
-
-
-TEST_SUITE = make_test_suite(Test_format_marcxml_file,
- Test_format_metadata,
- #Test_get_submission_status,
- #Test_format_submission_status,
- Test_swrCLIENTDATA_table)
- #Test_format_metadata_atom)
- #Test_list_submitted_resources)
- #Test_submission_process)
-
-if __name__ == "__main__":
- run_test_suite(TEST_SUITE)
-
diff --git a/modules/bibsword/lib/bibsword_config.py b/modules/bibsword/lib/bibsword_config.py
index 4df52e48f4..1ce97a6373 100644
--- a/modules/bibsword/lib/bibsword_config.py
+++ b/modules/bibsword/lib/bibsword_config.py
@@ -1,5 +1,7 @@
+# -*- coding: utf-8 -*-
+#
# This file is part of Invenio.
-# Copyright (C) 2010, 2011, 2013 CERN.
+# Copyright (C) 2010, 2011, 2012, 2013. 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -15,126 +17,76 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-'''
-Forward to ArXiv.org source code
-'''
+"""
+BibSword general configuration variables.
+"""
from invenio.bibformat_dblayer import get_tag_from_name
-
-#Maximal time to keep the stored XML Service doucment before reloading it in sec
-CFG_BIBSWORD_SERVICEDOCUMENT_UPDATE_TIME = 3600
-
-#Default submission status
-CFG_SUBMISSION_STATUS_SUBMITTED = "submitted"
-CFG_SUBMISSION_STATUS_PUBLISHED = "published"
-CFG_SUBMISSION_STATUS_ONHOLD = "onhold"
-CFG_SUBMISSION_STATUS_REMOVED = "removed"
-
-CFG_SUBMIT_ARXIV_INFO_MESSAGE = "Submitted from Invenio to arXiv by %s, on %s, as %s"
-CFG_DOCTYPE_UPLOAD_COLLECTION = 'PUSHED_TO_ARXIV'
-
-
-# report number:
-marc_tag_main_report_number = get_tag_from_name('primary report number')
-if marc_tag_main_report_number:
- CFG_MARC_REPORT_NUMBER = marc_tag_main_report_number
-else:
- CFG_MARC_REPORT_NUMBER = '037__a'
-
-# title:
-marc_tag_title = get_tag_from_name('title')
-if marc_tag_title:
- CFG_MARC_TITLE = marc_tag_title
-else:
- CFG_MARC_TITLE = '245__a'
-
-# author name:
-marc_tag_author = get_tag_from_name('first author name')
-if marc_tag_author:
- CFG_MARC_AUTHOR_NAME = marc_tag_author
-else:
- CFG_MARC_AUTHOR_NAME = '100__a'
-
-# author affiliation
-marc_tag_author_affiliation = get_tag_from_name('first author affiliation')
-if marc_tag_author_affiliation:
- CFG_MARC_AUTHOR_AFFILIATION = marc_tag_author_affiliation
-else:
- CFG_MARC_AUTHOR_AFFILIATION = '100__u'
-
-# contributor name:
-marc_tag_contributor_name = get_tag_from_name('additional author name')
-if marc_tag_contributor_name:
- CFG_MARC_CONTRIBUTOR_NAME = marc_tag_contributor_name
-else:
- CFG_MARC_CONTRIBUTOR_NAME = '700__a'
-
-# contributor affiliation:
-marc_tag_contributor_affiliation = get_tag_from_name('additional author affiliation')
-if marc_tag_contributor_affiliation:
- CFG_MARC_CONTRIBUTOR_AFFILIATION = marc_tag_contributor_affiliation
-else:
- CFG_MARC_CONTRIBUTOR_AFFILIATION = '700__u'
-
-# abstract:
-marc_tag_abstract = get_tag_from_name('main abstract')
-if marc_tag_abstract:
- CFG_MARC_ABSTRACT = marc_tag_abstract
-else:
- CFG_MARC_ABSTRACT = '520__a'
-
-# additional report number
-marc_tag_additional_report_number = get_tag_from_name('additional report number')
-if marc_tag_additional_report_number:
- CFG_MARC_ADDITIONAL_REPORT_NUMBER = marc_tag_additional_report_number
-else:
- CFG_MARC_ADDITIONAL_REPORT_NUMBER = '088__a'
-
-# doi
-marc_tag_doi = get_tag_from_name('doi')
-if marc_tag_doi:
- CFG_MARC_DOI = marc_tag_doi
-else:
- CFG_MARC_DOI = '909C4a'
-
-# journal code
-marc_tag_journal_ref_code = get_tag_from_name('journal code')
-if marc_tag_journal_ref_code:
- CFG_MARC_JOURNAL_REF_CODE = marc_tag_journal_ref_code
-else:
- CFG_MARC_JOURNAL_REF_CODE = '909C4c'
-
-# journal reference title
-marc_tag_journal_ref_title = get_tag_from_name('journal title')
-if marc_tag_journal_ref_title:
- CFG_MARC_JOURNAL_REF_TITLE = marc_tag_journal_ref_title
-else:
- CFG_MARC_JOURNAL_REF_TITLE = '909C4p'
-
-# journal reference page
-marc_tag_journal_ref_page = get_tag_from_name('journal page')
-if marc_tag_journal_ref_page:
- CFG_MARC_JOURNAL_REF_PAGE = marc_tag_journal_ref_page
-else:
- CFG_MARC_JOURNAL_REF_PAGE = '909C4v'
-
-# journal reference year
-marc_tag_journal_ref_year = get_tag_from_name('journal year')
-if marc_tag_journal_ref_year:
- CFG_MARC_JOURNAL_REF_YEAR = marc_tag_journal_ref_year
-else:
- CFG_MARC_JOURNAL_REF_YEAR = '909C4y'
-
-# comment
-marc_tag_comment = get_tag_from_name('comment')
-if marc_tag_comment:
- CFG_MARC_COMMENT = marc_tag_comment
-else:
- CFG_MARC_COMMENT = '500__a'
-
-# internal note field
-marc_tag_internal_note = get_tag_from_name('internal notes')
-if marc_tag_internal_note:
- CFG_MARC_RECORD_SUBMIT_INFO = marc_tag_internal_note
-else:
- CFG_MARC_RECORD_SUBMIT_INFO = '595__a'
+from invenio.urlutils import make_user_agent_string
+
+# Define the maximum number of contributors to be displayed and submitted
+CFG_BIBSWORD_MAXIMUM_NUMBER_OF_CONTRIBUTORS = 30
+
+# Define a custom timeout for urllib2 requests
+CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT = True
+CFG_BIBSWORD_DEFAULT_TIMEOUT = 20.0
+
+# The updated date format according to:
+# * http://tools.ietf.org/html/rfc4287#section-3.3
+# * https://dev.mysql.com/doc/refman/5.5/en/date-and-time-functions.html#function_date-format
+# NOTE: Specify the local timezone in the "+xx:yy" format.
+# If unknown, must be "Z".
+CFG_BIBSWORD_UPDATED_DATE_MYSQL_FORMAT = "%Y-%m-%dT%H:%i:%S"
+CFG_BIBSWORD_LOCAL_TIMEZONE = "+01:00"
+
+# The default atom entry mime-type for POST requests
+CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE = "application/atom+xml;type=entry"
+
+# Must be the same as in the client_servers/Makefile.am
+CFG_BIBSWORD_CLIENT_SERVERS_PATH = "bibsword_client_servers"
+
+CFG_BIBSWORD_USER_AGENT = make_user_agent_string("BibSword")
+
+CFG_BIBSWORD_MARC_FIELDS = {
+ 'rn': get_tag_from_name('primary report number') or '037__a',
+ 'title': get_tag_from_name('title') or '245__a',
+ 'author_name': get_tag_from_name('first author name') or '100__a',
+ 'author_affiliation':
+ get_tag_from_name('first author affiliation') or '100__u',
+ 'author_email': get_tag_from_name('first author email') or '859__f',
+ 'contributor_name':
+ get_tag_from_name('additional author name') or '700__a',
+ 'contributor_affiliation':
+ get_tag_from_name('additional author affiliation') or '700__u',
+ # 'abstract': get_tag_from_name('main abstract') or '520__a',
+ 'abstract': '520__a',
+ 'additional_rn': get_tag_from_name('additional report number') or '088__a',
+ # 'doi': get_tag_from_name('doi') or '909C4a',
+ 'doi': '909C4a',
+ # 'journal_code': get_tag_from_name('journal code') or '909C4c',
+ 'journal_code': '909C4v',
+ # 'journal_title': get_tag_from_name('journal title') or '909C4p',
+ 'journal_title': '909C4p',
+ # 'journal_page': get_tag_from_name('journal page') or '909C4v',
+ 'journal_page': '909C4c',
+ # 'journal_year': get_tag_from_name('journal year') or '909C4y',
+ 'journal_year': '909C4y',
+ 'comments': get_tag_from_name('comment') or '500__a',
+ 'internal_notes': get_tag_from_name('internal notes') or '595__a',
+}
+
+CFG_BIBSWORD_FILE_TYPES_MAPPING = {
+ 'application/atom+xml;type=entry': ['.xml'],
+ 'application/zip': ['.zip'],
+ 'application/xml': ['.wsdl', '.xpdl', '.rdf', '.xsl', '.xml', '.xsd'],
+ 'application/pdf': ['.pdf'],
+ 'application/postscript':
+ ['.ai', '.ps', '.eps', '.epsi', '.epsf', '.eps2', '.eps3'],
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+ ['.docx'],
+ 'text/xml': ['.xml'],
+ 'image/jpeg': ['.jpe', '.jpg', '.jpeg'],
+ 'image/jpg': ['.jpg'],
+ 'image/png': ['.png'],
+ 'image/gif': ['.gif'],
+}
diff --git a/modules/bibsword/lib/bibsword_webinterface.py b/modules/bibsword/lib/bibsword_webinterface.py
index 8bad0a4245..cc5202d3a8 100644
--- a/modules/bibsword/lib/bibsword_webinterface.py
+++ b/modules/bibsword/lib/bibsword_webinterface.py
@@ -1,8 +1,7 @@
-'''
-Forward to ArXiv.org source code
-'''
+# -*- coding: utf-8 -*-
+#
# This file is part of Invenio.
-# Copyright (C) 2010, 2011 CERN.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -18,496 +17,494 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-__revision__ = "$Id$"
-
-__lastupdated__ = """$Date$"""
-
-import os
-from invenio.access_control_engine import acc_authorize_action
-from invenio.config import CFG_SITE_URL, CFG_TMPDIR
-from invenio.webuser import page_not_authorized, collect_user_info
-from invenio.bibsword_client import perform_display_sub_status, \
- perform_display_server_list, \
- perform_display_collection_list, \
- perform_display_category_list, \
- perform_display_metadata, \
- perform_submit_record, \
- perform_display_server_infos, \
- list_remote_servers
-from invenio.webpage import page
-from invenio.messages import gettext_set_language
-from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory
-from invenio.websubmit_functions.Get_Recid import \
- get_existing_records_for_reportnumber
-from invenio.search_engine_utils import get_fieldvalues
-from invenio.bibsword_config import CFG_MARC_REPORT_NUMBER, CFG_MARC_ADDITIONAL_REPORT_NUMBER
-
-class WebInterfaceSword(WebInterfaceDirectory):
- """ Handle /bibsword set of pages."""
- _exports = ['', 'remoteserverinfos']
-
-
- def __init__(self, reqid=None):
- '''Initialize'''
- self.reqid = reqid
-
+"""
+BibSword Web Interface.
+"""
+
+from invenio.access_control_engine import(
+ acc_authorize_action
+)
+import invenio.bibsword_client as sword_client
+from invenio.config import(
+ CFG_SITE_LANG,
+ CFG_SITE_URL
+)
+from invenio.messages import(
+ gettext_set_language
+)
+from invenio.webinterface_handler import(
+ wash_urlargd,
+ WebInterfaceDirectory
+)
+from invenio.webpage import(
+ page
+)
+from invenio.webuser import(
+ getUid,
+ page_not_authorized
+)
- def __call__(self, req, form):
- errors = []
- warnings = []
- body = ''
- error_messages = []
+__lastupdated__ = """$Date$"""
- #***********************************************************************
- # Get values from the form
- #***********************************************************************
+class WebInterfaceSwordClient(WebInterfaceDirectory):
+ """Web interface for the BibSword client."""
+
+ _exports = [
+ "",
+ "servers",
+ "server_options",
+ "submissions",
+ "submission_options",
+ "submit",
+ "submit_step_1",
+ "submit_step_2",
+ "submit_step_3",
+ "submit_step_4",
+ ]
+
+ def submissions(self, req, form):
+ """Web interface for the existing submissions."""
+ # Check if the user has rights to manage the Sword client
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client/",
+ text=auth_message,
+ navtrail=""
+ )
argd = wash_urlargd(form, {
- 'ln' : (str, ''),
-
- # information of the state of the form submission
- 'status' : (str, ''),
- 'submit' : (str, ''),
- 'last_row' : (str, ''),
- 'first_row' : (str, ''),
- 'offset' : (int, ''),
- 'total_rows' : (str, ''),
-
- # mendatory informations
- 'id_record' : (str, ''),
- 'recid' : (int, 0),
- 'id_remote_server' : (str, ''),
- 'id_collection' : (str, ''),
- 'id_primary' : (str, ''),
- 'id_categories' : (list, []),
-
- 'id' : (str, ''),
- 'title' : (str, ''),
- 'summary' : (str, ''),
- 'author_name' : (str, ''),
- 'author_email' : (str, ''),
- 'contributor_name' : (list, []),
- 'contributor_email' : (list, []),
- 'contributor_affiliation' : (list, []),
-
- # optionnal informations
- 'comment' : (str, ''),
- 'doi' : (str, ''),
- 'type' : (str, ''),
- 'journal_refs' : (list, []),
- 'report_nos' : (list, []),
- 'media' : (list, []),
- 'new_media' : (str, ''),
- 'filename' : (str, '')
+ "ln": (str, CFG_SITE_LANG),
})
- # set language for i18n text auto generation
- _ = gettext_set_language(argd['ln'])
+ # Get the user ID
+ uid = getUid(req)
+
+ # Set language for i18n text auto generation
+ ln = argd["ln"]
+ _ = gettext_set_language(ln)
+
+ body = sword_client.perform_request_submissions(
+ ln
+ )
+
+ navtrail = """
+> %(label)s
+""" % {
+ 'CFG_SITE_URL': CFG_SITE_URL,
+ 'label': _("Sword Client"),
+ }
+
+ return page(
+ title=_("Submissions"),
+ body=body,
+ navtrail=navtrail,
+ lastupdated=__lastupdated__,
+ req=req,
+ language=ln
+ )
+
+ def submission_options(self, req, form):
+ """Web interface for the options on the submissions."""
+ # Check if the user has rights to manage the Sword client
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
+ argd = wash_urlargd(form, {
+ "option": (str, ""),
+ "action": (str, "submit"),
+ "server_id": (int, 0),
+ "status_url": (str, ""),
+ "ln": (str, CFG_SITE_LANG),
+ })
- #authentication
- (auth_code, auth_message) = self.check_credential(req)
+ if argd["option"] in ("update",):
+ option = argd["option"]
+ else:
+ option = ""
+
+ if argd["action"] in ("submit",):
+ action = argd["action"]
+ else:
+ action = ""
+
+ server_id = argd["server_id"]
+ status_url = argd["status_url"]
+ ln = argd["ln"]
+
+ (error, result) = sword_client.perform_request_submission_options(
+ option,
+ action,
+ server_id,
+ status_url,
+ ln
+ )
+
+ if error:
+ req.set_content_type("text/plain; charset=utf-8")
+ req.set_status("400")
+ req.send_http_header()
+ req.write("Error: {0}".format(error))
+ return
+
+ return result
+
+ def servers(self, req, form):
+ """Web interface for the available servers."""
+ # Check if the user has rights to manage the Sword client
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "manage_sword_client"
+ )
if auth_code != 0:
- return page_not_authorized(req=req, referer='/bibsword',
- text=auth_message, navtrail='')
-
-
- user_info = collect_user_info(req)
-
- #Build contributor tuples {name, email and affiliation(s)}
- contributors = []
- contributor_id = 0
- affiliation_id = 0
- for name in argd['contributor_name']:
- contributor = {}
- contributor['name'] = name
- contributor['email'] = argd['contributor_email'][contributor_id]
- contributor['affiliation'] = []
- is_last_affiliation = False
- while is_last_affiliation == False and \
- affiliation_id < len(argd['contributor_affiliation']):
- if argd['contributor_affiliation'][affiliation_id] == 'next':
- is_last_affiliation = True
- elif argd['contributor_affiliation'][affiliation_id] != '':
- contributor['affiliation'].append(\
- argd['contributor_affiliation'][affiliation_id])
- affiliation_id += 1
- contributors.append(contributor)
- contributor_id += 1
- argd['contributors'] = contributors
-
-
- # get the uploaded file(s) (if there is one)
- for key, formfields in form.items():
- if key == "new_media" and hasattr(formfields, "filename") and formfields.filename:
- filename = formfields.filename
- fp = open(os.path.join(CFG_TMPDIR, filename), "w")
- fp.write(formfields.file.read())
- fp.close()
- argd['media'].append(os.path.join(CFG_TMPDIR, filename))
- argd['filename'] = os.path.join(CFG_TMPDIR, filename)
-
- # Prepare navtrail
- navtrail = '''Admin Area ''' \
- % {'CFG_SITE_URL': CFG_SITE_URL}
-
- title = _("BibSword Admin Interface")
-
- #***********************************************************************
- # Display admin main page
- #***********************************************************************
-
- if argd['status'] == '' and argd['recid'] != '' and argd['id_remote_server'] != '':
- remote_servers = list_remote_servers(argd['id_remote_server'])
- if len(remote_servers) == 0:
- error_messages.append("No corresponding remote server could be found")
- (body, errors, warnings) = perform_display_server_list(
- error_messages,
- argd['id_record'])
- else:
- title = _("Export with BibSword: Step 2/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
- (body, errors, warnings) = perform_display_collection_list(
- argd['id_remote_server'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
- elif argd['status'] == '' or argd['submit'] == "Cancel":
- (body, errors, warnings) = perform_display_sub_status()
-
- elif argd['status'] == 'display_submission':
-
- if argd['submit'] == 'Refresh all':
- (body, errors, warnings) = \
- perform_display_sub_status(1, argd['offset'], "refresh_all")
-
- elif argd['submit'] == 'Select':
- first_row = 1
- (body, errors, warnings) = \
- perform_display_sub_status(first_row, argd['offset'])
-
- elif argd['submit'] == 'Next':
- first_row = int(argd['last_row']) + 1
- (body, errors, warnings) = \
- perform_display_sub_status(first_row, argd['offset'])
-
- elif argd['submit'] == 'Prev':
- first_row = int(argd['first_row']) - int(argd['offset'])
- (body, errors, warnings) = \
- perform_display_sub_status(first_row, argd['offset'])
-
- elif argd['submit'] == 'First':
- (body, errors, warnings) = \
- perform_display_sub_status(1, argd['offset'])
-
- elif argd['submit'] == 'Last':
- first_row = int(argd['total_rows']) - int(argd['offset']) + 1
- (body, errors, warnings) = \
- perform_display_sub_status(first_row, argd['offset'])
-
-
- #***********************************************************************
- # Select remote server
- #***********************************************************************
-
- # when the user validated the metadata, display
- elif argd['submit'] == 'New submission':
- title = _("Export with BibSword: Step 1/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
-
- (body, errors, warnings) = \
- perform_display_server_list(error_messages)
-
- # check if the user has selected a remote server
- elif argd['status'] == 'select_server':
- title = _("Export with BibSword: Step 1/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
-
- # check if given id_record exist and convert it in recid
- if argd['recid'] != 0:
- report_numbers = get_fieldvalues(argd['recid'], CFG_MARC_REPORT_NUMBER)
- report_numbers.extend(get_fieldvalues(argd['recid'], CFG_MARC_ADDITIONAL_REPORT_NUMBER))
- if report_numbers:
- argd['id_record'] = report_numbers[0]
-
- elif argd['id_record'] == '':
- error_messages.append("You must specify a report number")
-
- else:
- recids = \
- get_existing_records_for_reportnumber(argd['id_record'])
- if len(recids) == 0:
- error_messages.append(\
- "No document found with the given report number")
- elif len(recids) > 1:
- error_messages.append(\
- "Several documents have been found with given the report number")
- else:
- argd['recid'] = recids[0]
-
- if argd['id_remote_server'] in ['0', '']:
- error_messages.append("No remote server was selected")
-
- if not argd['id_remote_server'] in ['0', '']:
- # get the server's name and host
- remote_servers = list_remote_servers(argd['id_remote_server'])
- if len(remote_servers) == 0:
- error_messages.append("No corresponding remote server could be found")
- argd['id_remote_server'] = '0'
-
- if argd['id_remote_server'] in ['0', ''] or argd['recid'] == 0:
- (body, errors, warnings) = perform_display_server_list(
- error_messages,
- argd['id_record'])
-
- else:
- title = _("Export with BibSword: Step 2/4")
- (body, errors, warnings) = perform_display_collection_list(
- argd['id_remote_server'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
-
- #***********************************************************************
- # Select collection
- #***********************************************************************
-
- # check if the user wants to change the remote server
- elif argd['submit'] == 'Modify server':
- title = _("Export with BibSword: Step 1/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
- (body, errors, warnings) = \
- perform_display_server_list(error_messages, argd['id_record'])
-
- # check if the user has selected a collection
- elif argd['status'] == 'select_collection':
- title = _("Export with BibSword: Step 2/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL': CFG_SITE_URL}
- if argd['id_collection'] == '0':
- error_messages.append("No collection was selected")
- (body, errors, warnings) = perform_display_collection_list(
- argd['id_remote_server'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
- else:
- title = _("Export with BibSword: Step 3/4")
- (body, errors, warnings) = perform_display_category_list(
- argd['id_remote_server'],
- argd['id_collection'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
-
- #***********************************************************************
- # Select primary
- #***********************************************************************
-
- # check if the user wants to change the collection
- elif argd['submit'] == 'Modify collection':
- title = _("Export with BibSword: Step 2/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL': CFG_SITE_URL}
- (body, errors, warnings) = perform_display_collection_list(
- argd['id_remote_server'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
- # check if the user has selected a primary category
- elif argd['status'] == 'select_primary_category':
- title = _("Export with BibSword: Step 3/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
- if argd['id_primary'] == '0':
- error_messages.append("No primary category selected")
- (body, errors, warnings) = perform_display_category_list(
- argd['id_remote_server'],
- argd['id_collection'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
- else:
- title = _("Export with BibSword: Step 4/4")
- (body, errors, warnings) = perform_display_metadata(user_info,
- str(argd['id_remote_server']),
- str(argd['id_collection']),
- str(argd['id_primary']),
- argd['id_categories'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
- #***********************************************************************
- # Check record media and metadata
- #***********************************************************************
-
- # check if the user wants to change the collection
- elif argd['submit'] == 'Modify destination':
- title = _("Export with BibSword: Step 3/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
- (body, errors, warnings) = perform_display_category_list(
- argd['id_remote_server'],
- argd['id_collection'],
- argd['id_record'],
- argd['recid'],
- error_messages)
-
-
- # check if the metadata are complet and well-formed
- elif argd['status'] == 'check_submission':
- title = _("Export with BibSword: Step 4/4")
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
-
- if argd['submit'] == "Upload":
- error_messages.append("Media loaded")
-
- if argd['id'] == '':
- error_messages.append("Id is missing")
-
- if argd['title'] == '':
- error_messages.append("Title is missing")
-
- if argd['summary'] == '':
- error_messages.append("summary is missing")
- elif len(argd['summary']) < 25:
- error_messages.append("summary must have at least 25 character")
-
- if argd['author_name'] == '':
- error_messages.append("No submitter name specified")
-
- if argd['author_email'] == '':
- error_messages.append("No submitter email specified")
-
- if len(argd['contributors']) == 0:
- error_messages.append("No author specified")
-
- if len(error_messages) > 0:
-
- (body, errors, warnings) = perform_display_metadata(user_info,
- str(argd['id_remote_server']),
- str(argd['id_collection']),
- str(argd['id_primary']),
- argd['id_categories'],
- argd['id_record'],
- argd['recid'],
- error_messages,
- argd)
-
-
-
- else:
-
- title = _("Export with BibSword: Acknowledgement")
-
- navtrail += ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
-
- (body, errors, warnings) = perform_submit_record(user_info,
- str(argd['id_remote_server']),
- str(argd['id_collection']),
- str(argd['id_primary']),
- argd['id_categories'],
- argd['recid'],
- argd)
-
-
- # return of all the updated informations to be display
- return page(title = title,
- body = body,
- navtrail = navtrail,
- #uid = uid,
- lastupdated = __lastupdated__,
- req = req,
- language = argd['ln'],
- #errors = errors,
- warnings = warnings,
- navmenuid = "yourmessages")
-
-
- def remoteserverinfos(self, req, form):
- '''
- This method handle the /bibsword/remoteserverinfos call
- '''
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client/",
+ text=auth_message,
+ navtrail=""
+ )
argd = wash_urlargd(form, {
- 'ln' : (str, ''),
- 'id' : (str, '')
+ "ln": (str, CFG_SITE_LANG),
})
- #authentication
- (auth_code, auth_message) = self.check_credential(req)
+ # Get the user ID
+ uid = getUid(req)
+
+ # Set language for i18n text auto generation
+ ln = argd["ln"]
+ _ = gettext_set_language(ln)
+
+ body = sword_client.perform_request_servers(
+ ln
+ )
+
+ navtrail = """
+> %(label)s
+""" % {
+ 'CFG_SITE_URL': CFG_SITE_URL,
+ 'label': _("Sword Client"),
+ }
+
+ return page(
+ title=_("Servers"),
+ body=body,
+ navtrail=navtrail,
+ lastupdated=__lastupdated__,
+ req=req,
+ language=ln
+ )
+
+ def server_options(self, req, form):
+ """Web interface for the options on the available servers."""
+ # Check if the user has rights to manage the Sword client
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "manage_sword_client"
+ )
if auth_code != 0:
- return page_not_authorized(req=req, referer='/bibsword',
- text=auth_message, navtrail='')
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
+ argd = wash_urlargd(form, {
+ "option": (str, ""),
+ "action": (str, "submit"),
+ "server_id": (int, 0),
+ "sword_client_server_name": (str, ""),
+ "sword_client_server_engine": (str, ""),
+ "sword_client_server_username": (str, ""),
+ "sword_client_server_password": (str, ""),
+ "sword_client_server_email": (str, ""),
+ "sword_client_server_update_frequency": (str, ""),
+ "ln": (str, CFG_SITE_LANG),
+ })
- body = perform_display_server_infos(argd['id'])
+ if argd["option"] in ("add", "update", "modify", "delete"):
+ option = argd["option"]
+ else:
+ option = ""
+
+ if argd["action"] in ("prepare", "submit"):
+ action = argd["action"]
+ else:
+ action = ""
+
+ server_id = argd["server_id"]
+
+ server = (
+ argd["sword_client_server_name"],
+ argd["sword_client_server_engine"],
+ argd["sword_client_server_username"],
+ argd["sword_client_server_password"],
+ argd["sword_client_server_email"],
+ argd["sword_client_server_update_frequency"],
+ )
+
+ ln = argd["ln"]
+
+ (error, result) = sword_client.perform_request_server_options(
+ option,
+ action,
+ server_id,
+ server,
+ ln
+ )
+
+ if error:
+ req.set_content_type("text/plain; charset=utf-8")
+ req.set_status("400")
+ req.send_http_header()
+ req.write("Error: {0}".format(error))
+ return
+
+ return result
+
+ def submit(self, req, form):
+ """Submit a record using SWORD."""
+ # Check if the user has rights to manage the Sword client
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
- navtrail = ''' > ''' \
- '''SWORD Interface ''' % \
- {'CFG_SITE_URL' : CFG_SITE_URL}
+ argd = wash_urlargd(form, {
+ "record_id": (int, 0),
+ "server_id": (int, 0),
+ "ln": (str, CFG_SITE_LANG),
+ })
+ # Get the user ID
+ uid = getUid(req)
+
+ # Set language for i18n text auto generation
+ ln = argd["ln"]
+ _ = gettext_set_language(ln)
+
+ record_id = argd["record_id"]
+ server_id = argd["server_id"]
+
+ body = sword_client.perform_submit(
+ uid,
+ record_id,
+ server_id,
+ ln
+ )
+
+ navtrail = """
+> %(label)s
+""" % {
+ 'CFG_SITE_URL': CFG_SITE_URL,
+ 'label': _("Sword Client"),
+ }
+
+ return page(
+ title=_("Submit"),
+ body=body,
+ navtrail=navtrail,
+ lastupdated=__lastupdated__,
+ req=req,
+ language=ln
+ )
+
+ def submit_step_1(self, req, form):
+ """Process step 1 in the submission workflow."""
+ # Check if the user has adequate rights to run the bibsword client
+ # TODO: in a more advanced model, also check if the give user has
+ # rights to the current submission based on the user id and the
+ # submission id. It would get even more complicated if we
+ # introduced people that can approve specific submissions etc.
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
- # return of all the updated informations to be display
- return page(title = 'Remote server infos',
- body = body,
- navtrail = navtrail,
- #uid = uid,
- lastupdated = __lastupdated__,
- req = req,
- language = argd['ln'],
- errors = '',
- warnings = '',
- navmenuid = "yourmessages")
+ argd = wash_urlargd(form, {
+ 'sid': (str, ''),
+ 'server_id': (int, 0),
+ 'ln': (str, CFG_SITE_LANG),
+ })
+ sid = argd['sid']
+ server_id = argd['server_id']
+ ln = argd['ln']
+
+ return sword_client.perform_submit_step_1(
+ sid,
+ server_id,
+ ln
+ )
+
+ def submit_step_2(self, req, form):
+ """Process step 2 in the submission workflow."""
+ # Check if the user has adequate rights to run the bibsword client
+ # TODO: in a more advanced model, also check if the give user has
+ # rights to the current submission based on the user id and the
+ # submission id. It would get even more complicated if we
+ # introduced people that can approve specific submissions etc.
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
- def check_credential(self, req):
- '''
- This method check if the user has the right to get into this
- function
- '''
+ argd = wash_urlargd(form, {
+ 'sid': (str, ""),
+ 'collection_url': (str, ""),
+ 'ln': (str, CFG_SITE_LANG),
+ })
- auth_code, auth_message = acc_authorize_action(req, 'runbibswordclient')
- return (auth_code, auth_message)
+ sid = argd['sid']
+ collection_url = argd['collection_url']
+ ln = argd['ln']
+
+ return sword_client.perform_submit_step_2(
+ sid,
+ collection_url,
+ ln
+ )
+
+ def submit_step_3(self, req, form):
+ """Process step 3 in the submission workflow."""
+ # Check if the user has adequate rights to run the bibsword client
+ # TODO: in a more advanced model, also check if the give user has
+ # rights to the current submission based on the user id and the
+ # submission id. It would get even more complicated if we
+ # introduced people that can approve specific submissions etc.
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
+ argd = wash_urlargd(form, {
+ 'sid': (str, ""),
+ 'mandatory_category_url': (str, ""),
+ 'optional_categories_urls': (list, []),
+ 'ln': (str, CFG_SITE_LANG),
+ })
- index = __call__
+ sid = argd['sid']
+ ln = argd['ln']
+ mandatory_category_url = argd['mandatory_category_url']
+ optional_categories_urls = argd['optional_categories_urls']
+
+ return sword_client.perform_submit_step_3(
+ sid,
+ mandatory_category_url,
+ optional_categories_urls,
+ ln
+ )
+
+ def submit_step_4(self, req, form):
+ """Process step 4 in the submission workflow."""
+ # Check if the user has adequate rights to run the bibsword client
+ # TODO: in a more advanced model, also check if the give user has
+ # rights to the current submission based on the user id and the
+ # submission id. It would get even more complicated if we
+ # introduced people that can approve specific submissions etc.
+ auth_code, auth_message = acc_authorize_action(
+ req,
+ "run_sword_client"
+ )
+ if auth_code != 0:
+ return page_not_authorized(
+ req=req,
+ referer="/sword_client",
+ text=auth_message,
+ navtrail=""
+ )
+ argd = wash_urlargd(form, {
+ "sid": (str, ""),
+ "rn": (str, ""),
+ "additional_rn": (list, []),
+ "title": (str, ""),
+ "author_fullname": (str, ""),
+ "author_email": (str, ""),
+ "author_affiliation": (str, ""),
+ "abstract": (str, ""),
+ "contributor_fullname": (list, []),
+ "contributor_email": (list, []),
+ "contributor_affiliation": (list, []),
+ "files": (list, []),
+ "ln": (str, CFG_SITE_LANG),
+ })
+ sid = argd["sid"]
+ rn = argd["rn"]
+ additional_rn = argd["additional_rn"]
+ title = argd["title"]
+ author_fullname = argd["author_fullname"]
+ author_email = argd["author_email"]
+ author_affiliation = argd["author_affiliation"]
+ abstract = argd["abstract"]
+ contributor_fullname = argd["contributor_fullname"]
+ contributor_email = argd["contributor_email"]
+ contributor_affiliation = argd["contributor_affiliation"]
+ files_indexes = argd["files"]
+ ln = argd["ln"]
+
+ return sword_client.perform_submit_step_4(
+ sid,
+ (
+ rn,
+ additional_rn,
+ title,
+ author_fullname,
+ author_email,
+ author_affiliation,
+ abstract,
+ contributor_fullname,
+ contributor_email,
+ contributor_affiliation,
+ files_indexes
+ ),
+ ln
+ )
+
+ index = submissions
diff --git a/modules/bibsword/lib/client_servers/Makefile.am b/modules/bibsword/lib/client_servers/Makefile.am
new file mode 100644
index 0000000000..1455109aa5
--- /dev/null
+++ b/modules/bibsword/lib/client_servers/Makefile.am
@@ -0,0 +1,24 @@
+## This file is part of Invenio.
+## Copyright (C) 2010, 2011 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+pylibdir = $(libdir)/python/invenio/bibsword_client_servers
+
+pylib_DATA = arxiv_org.py __init__.py
+
+EXTRA_DIST = $(pylib_DATA)
+
+CLEANFILES = *~ *.tmp *.pyc
diff --git a/modules/bibsword/lib/client_servers/__init__.py b/modules/bibsword/lib/client_servers/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/modules/bibsword/lib/client_servers/arxiv_org.py b/modules/bibsword/lib/client_servers/arxiv_org.py
new file mode 100644
index 0000000000..f2896a0a58
--- /dev/null
+++ b/modules/bibsword/lib/client_servers/arxiv_org.py
@@ -0,0 +1,846 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+arXiv.org SWORD server.
+"""
+
+from cgi import escape
+from lxml import etree
+
+from invenio.bibsword_client_server import SwordClientServer
+
+__revision__ = "$Id$"
+
+_LOCAL_SETTINGS = {
+ 'realm': 'SWORD at arXiv',
+ 'uri': 'https://arxiv.org/sword-app/',
+ 'service_document_url': 'https://arxiv.org/sword-app/servicedocument',
+}
+
+CFG_ARXIV_ORG_VERBOSE = False
+CFG_ARXIV_ORG_DRY_RUN = False
+
+
+class ArxivOrg(SwordClientServer):
+ """
+ The arXiv.org SWORD server.
+ """
+
+ def __init__(self, settings):
+ """
+ Adds the arXiv.org-specific settings before running
+ SwordClientServer's constructor.
+ """
+
+ settings.update(_LOCAL_SETTINGS)
+ SwordClientServer.__init__(self, settings)
+
+ def _parse_service_document(
+ self,
+ service_document_raw
+ ):
+ """
+ Parses the raw service document (XML) into a python dictionary.
+ """
+
+ service_document_parsed = {}
+
+ try:
+ service_document_etree = etree.XML(service_document_raw)
+ except etree.XMLSyntaxError:
+ return service_document_parsed
+
+ # Get the version
+ version = service_document_etree.itertext(
+ tag="{http://purl.org/net/sword/}version"
+ ).next().strip()
+ service_document_parsed["version"] = version
+
+ # Get the maximum upload size
+ maximum_file_size_in_kilobytes = service_document_etree.itertext(
+ tag="{http://purl.org/net/sword/}maxUploadSize"
+ ).next().strip()
+ maximum_file_size_in_bytes = int(maximum_file_size_in_kilobytes)*1024
+ service_document_parsed["maxUploadSize"] = maximum_file_size_in_bytes
+
+ # Get the verbose
+ verbose = service_document_etree.itertext(
+ tag="{http://purl.org/net/sword/}verbose"
+ ).next().strip()
+ service_document_parsed["verbose"] = verbose
+
+ # Get the noOp
+ noOp = service_document_etree.itertext(
+ tag="{http://purl.org/net/sword/}noOp"
+ ).next().strip()
+ service_document_parsed["noOp"] = noOp
+
+ # Get the collections
+ service_document_parsed["collections"] = {}
+ collections = service_document_etree.iterdescendants(
+ tag='{http://www.w3.org/2007/app}collection'
+ )
+ for collection in collections:
+ collection_url = collection.get('href')
+ service_document_parsed["collections"][collection_url] = {}
+
+ # Get the collection title
+ collection_title = collection.itertext(
+ tag='{http://www.w3.org/2005/Atom}title'
+ ).next().strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "title"
+ ] = collection_title
+
+ # Get the collection accepted file types
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "accepts"
+ ] = []
+ collection_accepts = collection.iterdescendants(
+ tag='{http://www.w3.org/2007/app}accept'
+ )
+ for collection_accept in collection_accepts:
+ collection_accept_extensions = \
+ SwordClientServer._get_extension_for_file_type(
+ collection_accept.text.strip()
+ )
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "accepts"
+ ].extend(collection_accept_extensions)
+
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "accepts"
+ ] = tuple(
+ set(
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "accepts"
+ ]
+ )
+ )
+
+ # Get the collection policy
+ collection_policy = collection.itertext(
+ tag="{http://purl.org/net/sword/}collectionPolicy"
+ ).next().strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "policy"
+ ] = collection_policy
+
+ # Get the collection abstract
+ collection_abstract = collection.itertext(
+ tag="{http://purl.org/dc/terms/}abstract"
+ ).next().strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "abstract"
+ ] = collection_abstract
+
+ # Get the collection mediation
+ collection_mediation = collection.itertext(
+ tag="{http://purl.org/net/sword/}mediation"
+ ).next().strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "mediation"
+ ] = collection_mediation
+
+ # Get the collection treatment
+ collection_treatment = collection.itertext(
+ tag="{http://purl.org/net/sword/}treatment"
+ ).next().strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "treatment"
+ ] = collection_treatment
+
+ # Get the collection categories
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "categories"
+ ] = {}
+ collection_categories = collection.iterdescendants(
+ tag='{http://www.w3.org/2005/Atom}category'
+ )
+ for collection_category in collection_categories:
+ collection_category_term = collection_category.get(
+ 'term'
+ ).strip()
+ collection_category_scheme = collection_category.get(
+ 'scheme'
+ ).strip()
+ collection_category_label = collection_category.get(
+ 'label'
+ ).strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "categories"
+ ][
+ collection_category_term
+ ] = {
+ "label": collection_category_label,
+ "scheme": collection_category_scheme,
+ }
+
+ # Get the collection primary categories
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "primary_categories"
+ ] = {}
+ collection_primary_categories = collection.iterdescendants(
+ tag='{http://arxiv.org/schemas/atom/}primary_category'
+ )
+ for collection_primary_category in collection_primary_categories:
+ collection_primary_category_term = \
+ collection_primary_category.get(
+ 'term'
+ ).strip()
+ collection_primary_category_scheme = \
+ collection_primary_category.get(
+ 'scheme'
+ ).strip()
+ collection_primary_category_label = \
+ collection_primary_category.get(
+ 'label'
+ ).strip()
+ service_document_parsed[
+ "collections"
+ ][
+ collection_url
+ ][
+ "primary_categories"
+ ][
+ collection_primary_category_term
+ ] = {
+ "label": collection_primary_category_label,
+ "scheme": collection_primary_category_scheme,
+ }
+
+ return service_document_parsed
+
+ def get_collections(self):
+ """
+ """
+ return self.service_document_parsed.get("collections")
+
+ def get_categories(self, collection_url):
+ """
+ """
+ return {
+ "mandatory": self.service_document_parsed.get(
+ "collections",
+ ).get(
+ collection_url,
+ {}
+ ).get(
+ "primary_categories"
+ ),
+ "optional": self.service_document_parsed.get(
+ "collections"
+ ).get(
+ collection_url,
+ {}
+ ).get(
+ "categories"
+ ),
+ }
+
+ def get_accepted_file_types(self, collection_url):
+ """
+ """
+ return self.service_document_parsed.get(
+ "collections"
+ ).get(
+ collection_url,
+ {}
+ ).get(
+ "accepts"
+ )
+
+ def get_maximum_file_size(self):
+ """
+ Return size in bytes.
+ """
+ return self.service_document_parsed.get("maxUploadSize")
+
+ def _prepare_media(self, media):
+ """
+ TODO: this function should decide if and how to consolidate the files
+ depending on * the maximum file size
+ and * the accepted packaging options
+ """
+
+ return media
+
+ def _prepare_media_headers(self, file_info, metadata):
+ """
+ """
+
+ headers = SwordClientServer._prepare_media_headers(
+ self,
+ file_info,
+ metadata
+ )
+
+ headers['Host'] = 'arxiv.org'
+
+ if file_info['checksum']:
+ headers['Content-MD5'] = file_info['checksum']
+
+ # NOTE: It looks like media deposit only accepts the "X-On-Behalf-Of"
+ # header when both the author_name and author_email are present
+ # in the given format:
+ # "A. Scientist" .
+ (author_name, author_email, dummy) = metadata.get(
+ 'author',
+ (None, None, None)
+ )
+ if author_email:
+ if author_name and self._is_ascii(author_name):
+ headers['X-On-Behalf-Of'] = "\"%s\" <%s>" % (
+ author_name,
+ author_email
+ )
+ else:
+ headers['X-On-Behalf-Of'] = "%s" % (author_email,)
+
+ if CFG_ARXIV_ORG_VERBOSE:
+ headers['X-Verbose'] = 'True'
+
+ if CFG_ARXIV_ORG_DRY_RUN:
+ headers['X-No-Op'] = 'True'
+
+ return headers
+
+ def _prepare_media_response(self, response):
+ """
+ """
+
+ # NOTE: Could we get the media_link from the response headers?
+ # media_link = response.headers.get('Location')
+
+ try:
+ response_xml = response.read()
+ response.close()
+ response_etree = etree.XML(response_xml)
+ response_links = response_etree.findall(
+ '{http://www.w3.org/2005/Atom}link'
+ )
+ for response_link in response_links:
+ if response_link.attrib.get('rel') == 'edit-media':
+ media_link = response_link.attrib.get('href')
+ break
+ except:
+ media_link = None
+
+ return media_link
+
+ def _prepare_media_response_error(self, error):
+ """
+ """
+
+ try:
+ error_xml = error.read()
+ error.close()
+ error_etree = etree.XML(error_xml)
+ error_msg = error_etree.findtext(
+ '{http://www.w3.org/2005/Atom}summary'
+ )
+ if not error_msg:
+ raise Exception
+ return error_msg
+ except:
+ return "HTTP Error %s: %s" (error.code, error.msg)
+
+ def _prepare_metadata(self, metadata, media_response):
+ """
+ """
+
+ # Namespaces
+ # TODO: de-hardcode the namespaces
+ atom_ns = 'http://www.w3.org/2005/Atom'
+ atom_nsmap = {None: atom_ns}
+ arxiv_ns = 'http://arxiv.org/schemas/atom/'
+ arxiv_nsmap = {'arxiv': arxiv_ns}
+
+ # The root element of the atom entry
+ entry_element = etree.Element('entry', nsmap=atom_nsmap)
+
+ # The title element
+ # (SWORD/APP - arXiv.org mandatory)
+ # TODO: This is mandatory, we shouldn't check
+ # at this point if it's there or not.
+ title = metadata['title']
+ if title:
+ title_element = etree.Element('title')
+ title_element.text = escape(title, True).decode('utf-8')
+ entry_element.append(title_element)
+
+ # The id element
+ # (SWORD/APP mandatory)
+ # TODO: This is mandatory, we shouldn't check
+ # at this point if it's there or not.
+ recid = metadata['recid']
+ if recid:
+ id_element = etree.Element('id')
+ id_element.text = str(recid)
+ entry_element.append(id_element)
+
+ # The updated element
+ # (SWORD/APP mandatory)
+ # TODO: This is mandatory, we shouldn't check
+ # at this point if it's there or not.
+ modification_date = metadata['modification_date']
+ if modification_date:
+ updated_element = etree.Element('updated')
+ updated_element.text = modification_date
+ entry_element.append(updated_element)
+
+ # The author element
+ # (arXiv.org mandatory)
+ # NOTE: The author element is used for the authenticated user,
+ # i.e. the person who initiates the submission.
+ # The repeatable contributor element is used to specify the
+ # individual authors of the material being deposited to arXiv.
+ # At least one /contributor/email node must be present in order
+ # to inform arXiv of the email address of the primary contact
+ # author. If multiple /contributor/email nodes are found,
+ # the first will be used. Optionally the primary contact author's
+ # (name and) email address can also be specified in the
+ # X-On-Behalf-Of HTTP header extension, e.g.:
+ # X-On-Behalf-Of: "A. Scientist"
+ # (http://arxiv.org/help/submit_sword#Ingestion)
+ author_element = etree.Element('author')
+ author_name_element = etree.Element('name')
+ author_name_element.text = escape(self.username, True).decode('utf-8')
+ author_element.append(author_name_element)
+ author_email_element = etree.Element('email')
+ author_email_element.text = escape(self.email, True).decode('utf-8')
+ author_element.append(author_email_element)
+ entry_element.append(author_element)
+
+ # The contributors element(s)
+ # (arXiv.org mandatory)
+ # TODO: This is mandatory, we shouldn't check
+ # at this point if it's there or not.
+ # NOTE: At least one /contributor/email node must be present in order
+ # to inform arXiv of the email address of the primary contact
+ # author.
+ (author_name, author_email, author_affiliation) = metadata['author']
+ if author_name or author_email or author_affiliation:
+ contributor_element = etree.Element('contributor')
+ if author_name:
+ contributor_name_element = etree.Element('name')
+ contributor_name_element.text = escape(
+ author_name,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_name_element)
+ if author_email:
+ contributor_email_element = etree.Element('email')
+ contributor_email_element.text = escape(
+ author_email,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_email_element)
+ # TODO: Remove this hack with something more elegant.
+ else:
+ contributor_email_element = etree.Element('email')
+ contributor_email_element.text = escape(
+ self.email,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_email_element)
+ if author_affiliation:
+ contributor_affiliation_element = etree.Element(
+ '{%s}affiliation' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ contributor_affiliation_element.text = escape(
+ author_affiliation,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_affiliation_element)
+ entry_element.append(contributor_element)
+
+ contributors = metadata['contributors']
+ for (contributor_name,
+ contributor_email,
+ contributor_affiliation
+ ) in contributors:
+ contributor_element = etree.Element('contributor')
+ if contributor_name:
+ contributor_name_element = etree.Element('name')
+ contributor_name_element.text = escape(
+ contributor_name,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_name_element)
+ if contributor_email:
+ contributor_email_element = etree.Element('email')
+ contributor_email_element.text = escape(
+ contributor_email,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_email_element)
+ if contributor_affiliation:
+ contributor_affiliation_element = etree.Element(
+ '{%s}affiliation' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ contributor_affiliation_element.text = escape(
+ contributor_affiliation,
+ True
+ ).decode('utf-8')
+ contributor_element.append(contributor_affiliation_element)
+ entry_element.append(contributor_element)
+
+ # The content element
+ # (arXiv.org optional)
+ # NOTE: Contains or links to the complete content of the entry.
+ # Content must be provided if there is no alternate link,
+ # and should be provided if there is no summary.
+ # (http://www.atomenabled.org/developers/syndication/#recommendedEntryElements)
+
+ # The summary element
+ # (arXiv.org mandatory)
+ # TODO: This is mandatory, we shouldn't check
+ # at this point if it's there or not.
+ # The same goes for the minimum length of 20.
+ abstract = metadata['abstract']
+ if abstract:
+ # TODO: Replace this hack with something more elegant.
+ if len(abstract) < 20:
+ abstract += '.' * (20 - len(abstract))
+ summary_element = etree.Element('summary')
+ summary_element.text = escape(abstract, True).decode('utf-8')
+ entry_element.append(summary_element)
+
+ # The category element(s)
+ # (arXiv.org optional)
+ optional_categories = metadata['optional_categories']
+ for optional_category in optional_categories:
+ optional_category_element = etree.Element(
+ 'category',
+ term=optional_category['term'],
+ scheme=optional_category['scheme'],
+ label=optional_category['label'],
+ )
+ entry_element.append(optional_category_element)
+
+ # The primary_category element
+ # (arXiv.org mandatory)
+ mandatory_category = metadata['mandatory_category']
+ mandatory_category_element = etree.Element(
+ '{%s}primary_category' % (arxiv_ns,), nsmap=arxiv_nsmap,
+ term=mandatory_category['term'],
+ scheme=mandatory_category['scheme'],
+ label=mandatory_category['label'],
+ )
+ entry_element.append(mandatory_category_element)
+
+ # The report_no element(s)
+ # (arXiv.org optional)
+ rn = metadata['rn']
+ if rn:
+ report_no_element = etree.Element(
+ '{%s}report_no' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ report_no_element.text = escape(rn, True).decode('utf-8')
+ entry_element.append(report_no_element)
+
+ additional_rn = metadata['additional_rn']
+ for rn in additional_rn:
+ report_no_element = etree.Element(
+ '{%s}report_no' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ report_no_element.text = escape(rn, True).decode('utf-8')
+ entry_element.append(report_no_element)
+
+ # The doi element
+ # (arXiv.org optional)
+ doi = metadata['doi']
+ if doi:
+ doi_element = etree.Element(
+ '{%s}doi' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ doi_element.text = escape(doi, True).decode('utf-8')
+ entry_element.append(doi_element)
+
+ # The journal_ref element(s)
+ # (arXiv.org optional)
+ (
+ journal_code,
+ journal_title,
+ journal_page,
+ journal_year
+ ) = metadata['journal_info']
+ if journal_title:
+ journal_info = "%s" % (journal_title,)
+ if journal_code:
+ journal_info += ": %s" % (journal_code,)
+ if journal_year:
+ journal_info += " (%s)" % (journal_year,)
+ if journal_page:
+ journal_info += " pp. %s" % (journal_page,)
+ journal_ref_element = etree.Element(
+ '{%s}journal_ref' % (arxiv_ns,),
+ nsmap=arxiv_nsmap
+ )
+ journal_ref_element.text = escape(
+ journal_info, True
+ ).decode('utf-8')
+ entry_element.append(journal_ref_element)
+
+ # The comment element
+ # (arXiv.org optional)
+ # NOTE: The element contains the typical author
+ # comments found on most arXiv articles:
+ # "23 pages, 8 figures and 4 tables"
+ # TODO: How does this map to metadata['comments']
+ # and metadata['internal_notes']?
+
+ # The link element(s)
+ # (arXiv.org mandatory)
+ for file_response in media_response.itervalues():
+ if not file_response['error']:
+ link_element = etree.Element(
+ 'link',
+ href=file_response['msg'],
+ type=file_response['mime'],
+ rel='related'
+ )
+ entry_element.append(link_element)
+
+ prepared_metadata = etree.tostring(
+ entry_element,
+ xml_declaration=True,
+ encoding='utf-8',
+ pretty_print=True
+ )
+
+ return prepared_metadata
+
+ def _prepare_metadata_headers(self, metadata):
+ """
+ """
+
+ headers = SwordClientServer._prepare_metadata_headers(
+ self,
+ metadata
+ )
+
+ headers['Host'] = 'arxiv.org'
+
+ # NOTE: It looks like media deposit only accepts the "X-On-Behalf-Of"
+ # header when both the author_name and author_email are present
+ # in this format: "A. Scientist" .
+ (
+ author_name,
+ author_email,
+ dummy
+ ) = metadata.get('author', (None, None, None))
+ if author_email:
+ if author_name and self._is_ascii(author_name):
+ headers['X-On-Behalf-Of'] = "\"%s\" <%s>" % (
+ author_name,
+ author_email
+ )
+ else:
+ headers['X-On-Behalf-Of'] = "%s" % (author_email,)
+
+ if CFG_ARXIV_ORG_VERBOSE:
+ headers['X-Verbose'] = 'True'
+
+ if CFG_ARXIV_ORG_DRY_RUN:
+ headers['X-No-Op'] = 'True'
+
+ return headers
+
+ def _prepare_metadata_response(self, response):
+ """
+ """
+
+ links = {}
+
+ try:
+ response_xml = response.read()
+ response.close()
+ response_etree = etree.XML(response_xml)
+ response_links = response_etree.findall(
+ '{http://www.w3.org/2005/Atom}link'
+ )
+ for response_link in response_links:
+ if response_link.attrib.get('rel') == 'alternate':
+ links['alternate'] = response_link.attrib.get('href')
+ if response_link.attrib.get('rel') == 'edit':
+ links['edit'] = response_link.attrib.get('href')
+ except:
+ links['alternate'] = None
+ links['edit'] = None
+
+ return links
+
+ def _prepare_metadata_response_error(self, error):
+ """
+ """
+
+ try:
+ error_xml = error.read()
+ error.close()
+ error_etree = etree.XML(error_xml)
+ error_msg = error_etree.findtext(
+ '{http://www.w3.org/2005/Atom}summary'
+ )
+ if not error_msg:
+ raise Exception
+ return error_msg
+ except:
+ return "HTTP Error %s: %s" (error.code, error.msg)
+
+ def _prepare_response(self, media_response, metadata_response):
+ """
+ """
+
+ response = {}
+
+ # Update the general response with the metadata response
+ response.update(metadata_response)
+
+ # If there were errors with the media,
+ # also update the general response with the media response
+ if metadata_response['error'] and media_response:
+ for file_response in media_response.itervalues():
+ if file_response['error']:
+ response['media_response'] = media_response
+ break
+
+ return response
+
+ def _prepare_status_response(self, response):
+ """
+ """
+
+ status_and_error = {}
+
+ try:
+ response_xml = response.read()
+ response.close()
+ response_etree = etree.XML(response_xml)
+ response_status = response_etree.findall('status')
+ if response_status:
+ status_and_error['status'] = response_status[0].text
+ if status_and_error['status'] == "published":
+ response_arxiv_id = response_etree.findall('arxiv_id')
+ if response_arxiv_id:
+ status_and_error[
+ 'status'
+ ] = "{2} ".format(
+ "http://arxiv.org/abs/",
+ response_arxiv_id[0].text,
+ "published"
+ )
+ else:
+ status_and_error['status'] = None
+ response_error = response_etree.findall('error')
+ if response_error:
+ status_and_error['error'] = response_error[0].text
+ else:
+ status_and_error['error'] = None
+ except:
+ status_and_error['status'] = None
+ status_and_error['error'] = None
+
+ return status_and_error
+
+ @staticmethod
+ def _is_ascii(s):
+ """
+ """
+
+ try:
+ s.decode("ascii")
+ except:
+ return False
+ else:
+ return True
+
+# NOTE: in order to edit an existing submission:
+# * First all the media resources should be individually posted
+# or deposited packed together into a zip file.
+# * Then a metadata wrapper is PUT to the link with "rel=/edit/"
+# which was part of the atom entry response to the
+# original wrapper deposit.
+# ...
+# request = urllib2.Request('http://example.org', data='your_put_data')
+# request.get_method = lambda: 'PUT'
+# ...
+
+
+def arxiv_org(settings):
+ """
+ Instantiates the remote SWORD server.
+ """
+
+ return ArxivOrg(settings)
diff --git a/modules/bibupload/lib/batchuploader_engine.py b/modules/bibupload/lib/batchuploader_engine.py
index 90af6cac27..c0aa081831 100644
--- a/modules/bibupload/lib/batchuploader_engine.py
+++ b/modules/bibupload/lib/batchuploader_engine.py
@@ -60,7 +60,7 @@
from StringIO import StringIO
PERMITTED_MODES = ['-i', '-r', '-c', '-a', '-ir',
- '--insert', '--replace', '--correct', '--append']
+ '--insert', '--replace', '--correct', '--append', '--holdingpen']
_CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS_RE = re.compile(CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS)
diff --git a/modules/bibupload/lib/batchuploader_webinterface.py b/modules/bibupload/lib/batchuploader_webinterface.py
index 0f5f901f76..019320aac1 100644
--- a/modules/bibupload/lib/batchuploader_webinterface.py
+++ b/modules/bibupload/lib/batchuploader_webinterface.py
@@ -68,7 +68,8 @@ def legacyrobotupload(req, form):
return cli_upload(req, form.get('file', None), argd['mode'], argd['callback_url'], argd['nonce'], argd['special_treatment'])
if component == 'robotupload':
- if path and path[0] in ('insert', 'replace', 'correct', 'append', 'insertorreplace'):
+ if path and path[0] in ('insert', 'replace', 'correct', 'append',
+ 'insertorreplace', 'holdingpen'):
return restupload, None
else:
return legacyrobotupload, None
diff --git a/modules/bibupload/lib/bibupload.py b/modules/bibupload/lib/bibupload.py
index 5a056b147b..d80213d09e 100644
--- a/modules/bibupload/lib/bibupload.py
+++ b/modules/bibupload/lib/bibupload.py
@@ -80,6 +80,9 @@
record_extract_dois, \
record_has_field, \
records_identical, \
+ filter_field_instances, \
+ create_field, \
+ record_add_fields, \
record_drop_duplicate_fields
from invenio.search_engine import get_record, record_exists, search_pattern
from invenio.dateutils import convert_datestruct_to_datetext
@@ -98,8 +101,6 @@
bibdocfile_url_p, CFG_BIBDOCFILE_AVAILABLE_FLAGS, guess_format_from_url, \
BibRelation, MoreInfo
-from invenio.search_engine import search_pattern
-
from invenio.bibupload_revisionverifier import RevisionVerifier, \
InvenioBibUploadConflictingRevisionsError, \
InvenioBibUploadInvalidRevisionError, \
@@ -499,6 +500,14 @@ def bibupload(record, opt_mode=None, opt_notimechange=0, oai_rec_id="", pretend=
affected_tags['856'] = [('4', ' ')]
elif ('4', ' ') not in affected_tags['856']:
affected_tags['856'].append(('4', ' '))
+ # 540 and 542 fields have been replaced too
+ # TODO this may be improved with a function that checks
+ # if those fields have been replaced with different values
+ # than before
+ if '540' not in affected_tags:
+ affected_tags['540'] = [(' ', ' ')]
+ if '542' not in affected_tags:
+ affected_tags['542'] = [(' ', ' ')]
write_message(" -Modified field list updated with FFT details: %s" % str(affected_tags), verbose=2)
except Exception, e:
register_exception(alert_admin=True)
@@ -1435,6 +1444,7 @@ def get_bibdocfile_managed_info():
for afile in latest_files:
url = afile.get_url()
ret[url] = {'u': url}
+ ret[url]['8'] = str(afile.get_bibdocid())
description = afile.get_description()
comment = afile.get_comment()
subformat = afile.get_subformat()
@@ -1449,6 +1459,95 @@ def get_bibdocfile_managed_info():
return ret
+ def merge_copyright_and_license_info_marc():
+ """
+ Removes old copyright and license fields from MARC and replaces them
+ with new fields from BibDocs
+ """
+ copyright_fields = record_get_field_instances(record, '542', '%', '%')
+ license_fields = record_get_field_instances(record, '540', '%', '%')
+ # remove fields that have subfield $8 (we will recreate that fields
+ # based on the data from BibDocs)
+ copyrights_to_remove = filter_field_instances(copyright_fields, '8', '.*', 'r')
+ licenses_to_remove = filter_field_instances(license_fields, '8', '.*', 'r')
+ # Store copyrights and licenses in separate list
+ for field in copyrights_to_remove:
+ write_message("Removing %s field" % (field, ), verbose=9)
+ record_delete_field(record, '542', '', '', field[4])
+ for field in licenses_to_remove:
+ write_message("Removing %s field" % (field, ), verbose=9)
+ record_delete_field(record, '540', '', '', field[4])
+
+ copyright_fields = create_copyright_fields()
+ write_message("Adding 542 fields: %s" % (copyright_fields, ), verbose=9)
+ record_add_fields(record, '542', copyright_fields)
+ license_fields = create_license_fields()
+ write_message("Adding 540 fields: %s" % (license_fields, ), verbose=9)
+ record_add_fields(record, '540', license_fields)
+
+ def create_copyright_fields():
+ """
+ Reads copyright from each BibDoc in BibRecDoc, uses them to create
+ a list of new fields, that can be added to the MARC, using
+ create_field() function and returns that list
+ """
+ fields = []
+ bibrecdocs = BibRecDocs(rec_id)
+ bibdocs = bibrecdocs.list_bibdocs()
+ for bibdoc in bibdocs:
+ bibdocid = bibdoc.get_id()
+ subfields = []
+ subfields.append(('8', str(bibdocid)))
+ copyright = bibdoc.get_copyright()
+ copyright_holder = copyright.get('copyright_holder')
+ copyright_date = copyright.get('copyright_date')
+ copyright_message = copyright.get('copyright_message')
+ copyright_holder_contact = copyright.get('copyright_holder_contact')
+ if copyright_holder:
+ subfields.append(('d', copyright_holder))
+ if copyright_holder_contact:
+ subfields.append(('e', copyright_holder_contact))
+ if copyright_message:
+ subfields.append(('f', copyright_message))
+ if copyright_date:
+ subfields.append(('g', copyright_date))
+ if len(subfields) > 1:
+ # Len > 1 means that we actually have something more that just
+ # bibdocid in the subfield, so we can add this subfield to the MARC
+ fields.append(create_field(subfields))
+
+ return fields
+
+ def create_license_fields():
+ """
+ Reads license from each BibDoc in BibRecDoc, uses them to create
+ a list of new fields, that can be added to the MARC, using
+ create_field() function and returns that list
+ """
+ fields = []
+ bibrecdocs = BibRecDocs(rec_id)
+ bibdocs = bibrecdocs.list_bibdocs()
+ for bibdoc in bibdocs:
+ bibdocid = bibdoc.get_id()
+ subfields = []
+ subfields.append(('8', str(bibdocid)))
+ license = bibdoc.get_license()
+ license_name = license.get('license')
+ license_url = license.get('license_url')
+ license_body = license.get('license_body')
+ if license_name:
+ subfields.append(('a', license_name))
+ if license_url:
+ subfields.append(('b', license_url))
+ if license_body:
+ subfields.append(('u', license_body))
+ if len(subfields) > 1:
+ # Len > 1 means that we actually have something more that just
+ # bibdocid in the subfield, so we can add this subfield to the MARC
+ fields.append(create_field(subfields))
+
+ return fields
+
write_message("Synchronizing MARC of recid '%s' with:\n%s" % (rec_id, record), verbose=9)
tags856s = record_get_field_instances(record, '856', '%', '%')
write_message("Original 856%% instances: %s" % tags856s, verbose=9)
@@ -1489,6 +1588,9 @@ def get_bibdocfile_managed_info():
subfields.sort()
record_add_field(record, '856', '4', ' ', subfields=subfields)
+ # Synchronize copyright and license fields in MARC from BibDocs
+ write_message("Synchronizing 542 and 540 fields with BibDocs", verbose=9)
+ merge_copyright_and_license_info_marc()
write_message('Final record: %s' % record, verbose=9)
return record
@@ -1708,34 +1810,37 @@ def elaborate_fft_tags(record, rec_id, mode, pretend=False,
"""
# Let's define some handy sub procedure.
- def _add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, modification_date, pretend=False):
+ def _add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, modification_date, pretend=False):
"""Adds a new format for a given bibdoc. Returns True when everything's fine."""
- write_message('Add new format to %s url: %s, format: %s, docname: %s, doctype: %s, newname: %s, description: %s, comment: %s, flags: %s, modification_date: %s' % (repr(bibdoc), url, docformat, docname, doctype, newname, description, comment, flags, modification_date), verbose=9)
+ write_message('Add new format to %s url: %s, format: %s, docname: %s, doctype: %s, newname: %s, description: %s, comment: %s, copyright: %s, license: %s, flags: %s, modification_date: %s' % (repr(bibdoc), url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, modification_date), verbose=9)
try:
- if not url: # Not requesting a new url. Just updating comment & description
- return _update_description_and_comment(bibdoc, docname, docformat, description, comment, flags, pretend=pretend)
+ if not url: # Not requesting a new url. Just updating comment, description, copyright and license
+ return (_update_description_and_comment(bibdoc, docname, docformat, description, comment, flags, pretend=pretend) and
+ _update_copyright_and_license(bibdoc, docname, copyright, license))
try:
if not pretend:
- bibdoc.add_file_new_format(url, description=description, comment=comment, flags=flags, modification_date=modification_date)
+ bibdoc.add_file_new_format(url, description=description, comment=comment, copyright=copyright, license=license, flags=flags, modification_date=modification_date)
except StandardError, e:
- write_message("('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s') not inserted because format already exists (%s)." % (url, docformat, docname, doctype, newname, description, comment, flags, modification_date, e), stream=sys.stderr)
+ write_message("('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s') not inserted because format already exists (%s)." % (url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, modification_date, e), stream=sys.stderr)
raise
except Exception, e:
write_message("ERROR: in adding '%s' as a new format because of: %s" % (url, e), stream=sys.stderr)
raise
return True
- def _add_new_version(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, modification_date, pretend=False):
+ def _add_new_version(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, modification_date, pretend=False):
"""Adds a new version for a given bibdoc. Returns True when everything's fine."""
- write_message('Add new version to %s url: %s, format: %s, docname: %s, doctype: %s, newname: %s, description: %s, comment: %s, flags: %s' % (repr(bibdoc), url, docformat, docname, doctype, newname, description, comment, flags), verbose=9)
+ write_message('Add new version to %s url: %s, format: %s, docname: %s, doctype: %s, newname: %s, description: %s, comment: %s, copyright: %s, license: %s, flags: %s' % (repr(bibdoc), url, docformat, docname, doctype, newname, description, comment, copyright, license, flags), verbose=9)
try:
if not url:
- return _update_description_and_comment(bibdoc, docname, docformat, description, comment, flags, pretend=pretend)
+ # Not requesting a new url. Just updating comment, description, copyright and license
+ return (_update_description_and_comment(bibdoc, docname, docformat, description, comment, flags, pretend=pretend) and
+ _update_copyright_and_license(bibdoc, docname, copyright, license))
try:
if not pretend:
- bibdoc.add_file_new_version(url, description=description, comment=comment, flags=flags, modification_date=modification_date)
+ bibdoc.add_file_new_version(url, description=description, comment=comment, copyright=copyright, license=license, flags=flags, modification_date=modification_date)
except StandardError, e:
- write_message("('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s') not inserted because '%s'." % (url, docformat, docname, doctype, newname, description, comment, flags, modification_date, e), stream=sys.stderr)
+ write_message("('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s') not inserted because '%s'." % (url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, modification_date, e), stream=sys.stderr)
raise
except Exception, e:
write_message("ERROR: in adding '%s' as a new version because of: %s" % (url, e), stream=sys.stderr)
@@ -1759,6 +1864,16 @@ def _update_description_and_comment(bibdoc, docname, docformat, description, com
raise
return True
+ def _update_copyright_and_license(bibdoc, docname, copyright, license):
+ write_message('Just updating copyright and license for %s with copyright %s, license %s' % (docname, copyright, license), verbose=9)
+ try:
+ bibdoc.set_copyright(copyright)
+ bibdoc.set_license(license)
+ except StandardError, e :
+ write_message("('%s', '%s', '%s') copyright and license not updated because '%s'." % (docname, copyright, license, e))
+ raise
+ return True
+
def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
if not mode in ('correct', 'append', 'replace_or_insert', 'replace', 'insert'):
#print "exited because the mode is incorrect"
@@ -1949,7 +2064,48 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
else:
restriction = KEEP_OLD_VALUE
-
+ # Let's discover the copyright
+ copyright = field_get_subfield_values(fft, 'c')
+ if copyright != []:
+ # copyright information should be separated with ||
+ copyright = copyright[0].split("||")
+ # There should be 4 pieces of information about the copyright:
+ # copyright holder, date, message and holder contact
+ # If some of them is missing, fill the list with empty items
+ if len(copyright) < 4:
+ copyright.extend((4-len(copyright))*[''])
+ # Create copyright dictionary
+ copyright = {'copyright_holder' : copyright[0],
+ 'copyright_date' : copyright[1],
+ 'copyright_message' : copyright[2],
+ 'copyright_holder_contact' : copyright[3]}
+ else:
+ if mode == 'correct' and doctype != 'FIX-MARC':
+ ## See comment on description
+ copyright = ''
+ else:
+ copyright = KEEP_OLD_VALUE
+
+ # Let's discover the license
+ license = field_get_subfield_values(fft, 'l')
+ if license != []:
+ # license information should be separated with ||
+ license = license[0].split("||")
+ # There should be 3 pieces of information about the license:
+ # license itself, url and license body
+ # If some of them is missing, fill the list with empty items
+ if len(license) < 3:
+ license.extend((3-len(license))*[''])
+ # Create license dictionary
+ license = {'license' : license[0],
+ 'license_url' : license[1],
+ 'license_body' : license[2]}
+ else:
+ if mode == 'correct' and doctype != 'FIX-MARC':
+ ## See comment on description
+ license = ''
+ else:
+ license = KEEP_OLD_VALUE
document_moreinfo = _get_subfield_value(fft, 'w')
version_moreinfo = _get_subfield_value(fft, 'p')
version_format_moreinfo = _get_subfield_value(fft, 'b')
@@ -1983,20 +2139,20 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
raise StandardError, "fft '%s' specifies a different restriction from previous fft with docname '%s'" % (str(fft), name)
if version2 != version:
raise StandardError, "fft '%s' specifies a different version than the previous fft with docname '%s'" % (str(fft), name)
- for (dummyurl2, format2, dummydescription2, dummycomment2, dummyflags2, dummytimestamp2) in urls:
+ for (dummyurl2, format2, dummydescription2, dummycomment2, dummycopyright2, dummylicense2, dummyflags2, dummytimestamp2) in urls:
if docformat == format2:
raise StandardError, "fft '%s' specifies a second file '%s' with the same format '%s' from previous fft with docname '%s'" % (str(fft), url, docformat, name)
if url or docformat:
- urls.append((url, docformat, description, comment, flags, timestamp))
+ urls.append((url, docformat, description, comment, copyright, license, flags, timestamp))
if icon:
- urls.append((icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, flags, timestamp))
+ urls.append((icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, copyright, license, flags, timestamp))
else:
if url or docformat:
- docs[name] = (doctype, newname, restriction, version, [(url, docformat, description, comment, flags, timestamp)], [document_moreinfo, version_moreinfo, version_format_moreinfo, format_moreinfo], bibdoc_tmpid, bibdoc_tmpver)
+ docs[name] = (doctype, newname, restriction, version, [(url, docformat, description, comment, copyright, license, flags, timestamp)], [document_moreinfo, version_moreinfo, version_format_moreinfo, format_moreinfo], bibdoc_tmpid, bibdoc_tmpver)
if icon:
- docs[name][4].append((icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, flags, timestamp))
+ docs[name][4].append((icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, copyright, license, flags, timestamp))
elif icon:
- docs[name] = (doctype, newname, restriction, version, [(icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, flags, timestamp)], [document_moreinfo, version_moreinfo, version_format_moreinfo, format_moreinfo], bibdoc_tmpid, bibdoc_tmpver)
+ docs[name] = (doctype, newname, restriction, version, [(icon, icon[len(file_strip_ext(icon)):] + ';icon', description, comment, copyright, license, flags, timestamp)], [document_moreinfo, version_moreinfo, version_format_moreinfo, format_moreinfo], bibdoc_tmpid, bibdoc_tmpver)
else:
docs[name] = (doctype, newname, restriction, version, [], [document_moreinfo, version_moreinfo, version_format_moreinfo, format_moreinfo], bibdoc_tmpid, bibdoc_tmpver)
@@ -2020,7 +2176,7 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
bibdoc = None
new_revision_needed = False
- for url, docformat, description, comment, flags, timestamp in urls:
+ for url, docformat, description, comment, copyright, license, flags, timestamp in urls:
if url:
try:
downloaded_url = download_url(url, docformat)
@@ -2029,27 +2185,27 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
write_message("ERROR: in downloading '%s' because of: %s" % (url, err), stream=sys.stderr)
raise
if mode == 'correct' and bibdoc is not None and not new_revision_needed:
- downloaded_urls.append((downloaded_url, docformat, description, comment, flags, timestamp))
+ downloaded_urls.append((downloaded_url, docformat, description, comment, copyright, license, flags, timestamp))
if not bibrecdocs.check_file_exists(downloaded_url, docformat):
new_revision_needed = True
else:
write_message("WARNING: %s is already attached to bibdoc %s for recid %s" % (url, docname, rec_id), stream=sys.stderr)
elif mode == 'append' and bibdoc is not None:
if not bibrecdocs.check_file_exists(downloaded_url, docformat):
- downloaded_urls.append((downloaded_url, docformat, description, comment, flags, timestamp))
+ downloaded_urls.append((downloaded_url, docformat, description, comment, copyright, license, flags, timestamp))
else:
write_message("WARNING: %s is already attached to bibdoc %s for recid %s" % (url, docname, rec_id), stream=sys.stderr)
else:
- downloaded_urls.append((downloaded_url, docformat, description, comment, flags, timestamp))
+ downloaded_urls.append((downloaded_url, docformat, description, comment, copyright, license, flags, timestamp))
else:
- downloaded_urls.append(('', docformat, description, comment, flags, timestamp))
+ downloaded_urls.append(('', docformat, description, comment, copyright, license, flags, timestamp))
if mode == 'correct' and bibdoc is not None and not new_revision_needed:
## Since we don't need a new revision (because all the files
## that are being uploaded are different)
## we can simply remove the urls but keep the other information
write_message("No need to add a new revision for docname %s for recid %s" % (docname, rec_id), verbose=2)
- docs[docname] = (doctype, newname, restriction, version, [('', docformat, description, comment, flags, timestamp) for (dummy, docformat, description, comment, flags, timestamp) in downloaded_urls], more_infos, bibdoc_tmpid, bibdoc_tmpver)
- for downloaded_url, dummy, dummy, dummy, dummy, dummy in downloaded_urls:
+ docs[docname] = (doctype, newname, restriction, version, [('', docformat, description, comment, copyright, license, flags, timestamp) for (dummy, docformat, description, comment, copyright, license, flags, timestamp) in downloaded_urls], more_infos, bibdoc_tmpid, bibdoc_tmpver)
+ for downloaded_url, _, _, _, _, _, _, _ in downloaded_urls:
## Let's free up some space :-)
if downloaded_url and os.path.exists(downloaded_url):
os.remove(downloaded_url)
@@ -2082,8 +2238,8 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
except Exception, e:
write_message("('%s', '%s', '%s') not inserted because: '%s'." % (doctype, newname, urls, e), stream=sys.stderr)
raise e
- for (url, docformat, description, comment, flags, timestamp) in urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp, pretend=pretend))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp, pretend=pretend))
elif mode == 'replace_or_insert': # to be thought as correct_or_insert
try:
bibdoc = bibrecdocs.get_bibdoc(docname)
@@ -2143,18 +2299,18 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
# bump the version by pushing the first new file
# then pushing the other files.
if urls:
- (first_url, first_format, first_description, first_comment, first_flags, first_timestamp) = urls[0]
+ (first_url, first_format, first_description, first_comment, first_copyright, first_license, first_flags, first_timestamp) = urls[0]
other_urls = urls[1:]
- assert(_add_new_version(bibdoc, first_url, first_format, docname, doctype, newname, first_description, first_comment, first_flags, first_timestamp, pretend=pretend))
- for (url, docformat, description, comment, flags, timestamp) in other_urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp, pretend=pretend))
+ assert(_add_new_version(bibdoc, first_url, first_format, docname, doctype, newname, first_description, first_comment, first_copyright, first_license, first_flags, first_timestamp, pretend=pretend))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in other_urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp, pretend=pretend))
## Let's refresh the list of bibdocs.
if not found_bibdoc:
if not pretend:
bibdoc = bibrecdocs.add_bibdoc(doctype, newname)
bibdoc.set_status(restriction)
- for (url, docformat, description, comment, flags, timestamp) in urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp))
elif mode == 'correct':
try:
bibdoc = bibrecdocs.get_bibdoc(docname)
@@ -2196,7 +2352,7 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
pass
elif doctype == 'DELETE-FILE':
if urls:
- for (url, docformat, description, comment, flags, timestamp) in urls:
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
if not pretend:
bibdoc.delete_file(docformat, version)
elif doctype == 'REVERT':
@@ -2214,11 +2370,11 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
if not pretend:
bibdoc.change_doctype(doctype)
if urls:
- (first_url, first_format, first_description, first_comment, first_flags, first_timestamp) = urls[0]
+ (first_url, first_format, first_description, first_comment, first_copyright, first_license, first_flags, first_timestamp) = urls[0]
other_urls = urls[1:]
- assert(_add_new_version(bibdoc, first_url, first_format, docname, doctype, newname, first_description, first_comment, first_flags, first_timestamp, pretend=pretend))
- for (url, docformat, description, comment, flags, timestamp) in other_urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp, pretend=pretend))
+ assert(_add_new_version(bibdoc, first_url, first_format, docname, doctype, newname, first_description, first_comment, first_copyright, first_license, first_flags, first_timestamp, pretend=pretend))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in other_urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp, pretend=pretend))
if not found_bibdoc:
if doctype in ('PURGE', 'DELETE', 'EXPUNGE', 'FIX-ALL', 'FIX-MARC', 'DELETE-FILE', 'REVERT'):
write_message("('%s', '%s', '%s') not performed because '%s' docname didn't existed." % (doctype, newname, urls, docname), stream=sys.stderr)
@@ -2227,8 +2383,8 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
if not pretend:
bibdoc = bibrecdocs.add_bibdoc(doctype, newname)
bibdoc.set_status(restriction)
- for (url, docformat, description, comment, flags, timestamp) in urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp))
elif mode == 'append':
found_bibdoc = False
try:
@@ -2237,15 +2393,15 @@ def _process_document_moreinfos(more_infos, docname, version, docformat, mode):
except InvenioBibDocFileError:
found_bibdoc = False
else:
- for (url, docformat, description, comment, flags, timestamp) in urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp, pretend=pretend))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp, pretend=pretend))
if not found_bibdoc:
try:
if not pretend:
bibdoc = bibrecdocs.add_bibdoc(doctype, docname)
bibdoc.set_status(restriction)
- for (url, docformat, description, comment, flags, timestamp) in urls:
- assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, flags, timestamp))
+ for (url, docformat, description, comment, copyright, license, flags, timestamp) in urls:
+ assert(_add_new_format(bibdoc, url, docformat, docname, doctype, newname, description, comment, copyright, license, flags, timestamp))
except Exception, e:
register_exception()
write_message("('%s', '%s', '%s') not appended because: '%s'." % (doctype, newname, urls, e), stream=sys.stderr)
diff --git a/modules/bibupload/lib/bibupload_regression_tests.py b/modules/bibupload/lib/bibupload_regression_tests.py
index c8af4285b4..b0f68fa884 100644
--- a/modules/bibupload/lib/bibupload_regression_tests.py
+++ b/modules/bibupload/lib/bibupload_regression_tests.py
@@ -3869,6 +3869,7 @@ def test_simple_fft_insert(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -3885,6 +3886,12 @@ def test_simple_fft_insert(self):
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -3920,6 +3927,7 @@ def test_fft_insert_with_valid_embargo(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -3936,6 +3944,12 @@ def test_fft_insert_with_valid_embargo(self):
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -3975,6 +3989,7 @@ def test_fft_insert_with_expired_embargo(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -3994,6 +4009,12 @@ def test_fft_insert_with_expired_embargo(self):
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4051,7 +4072,8 @@ def test_exotic_format_fft_append(self):
jekyll@cds.cern.ch
- 4
+ 987654321
+ 4
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z
@@ -4082,6 +4104,13 @@ def test_exotic_format_fft_append(self):
recs = bibupload.xml_marc_to_records(testrec_to_append)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='append')[0]
self.check_record_consistency(recid)
+
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4135,12 +4164,14 @@ def test_detailed_fft_insert(self):
%(siteurl)s/img/site_logo.gif
SuperMain
+ Copyright Holder||1234||CopyrightMessage||jekyll@cds.cern.ch
This is a description
This is a comment
CIDIESSE
%(siteurl)s/img/rss.png
+ License||www.license.url.com||license body
SuperMain
.jpeg
This is a description
@@ -4159,13 +4190,28 @@ def test_detailed_fft_insert(self):
Test, John
Test University
+
+ 9876543211
+ License
+ www.license.url.com
+ license body
+
+
+ 9876543211
+ Copyright Holder
+ jekyll@cds.cern.ch
+ CopyrightMessage
+ 1234
+
+ 9876543211
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif
This is a description
This is a comment
+ 9876543212
530
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg
This is a description
@@ -4186,6 +4232,16 @@ def test_detailed_fft_insert(self):
str(recid))
testrec_expected_url2 = testrec_expected_url1.replace('123456789',
str(recid))
+
+ # get the bibdocid of latest file inserted
+ bibdocids = [x.get_bibdocid() for x in BibRecDocs(recid).list_latest_files()]
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(min(bibdocids)))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(max(bibdocids)))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4237,10 +4293,12 @@ def test_simple_fft_insert_with_restriction(self):
jekyll@cds.cern.ch
+ 9876543211
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
+ 9876543212
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
@@ -4271,6 +4329,16 @@ def test_simple_fft_insert_with_restriction(self):
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
+
+ # get the bibdocids of latest file inserted
+ bibdocids = [x.get_bibdocid() for x in BibRecDocs(recid).list_latest_files()]
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(min(bibdocids)))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(max(bibdocids)))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4311,10 +4379,12 @@ def test_simple_fft_insert_with_icon(self):
Test University
+ 9876543211
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
+ 9876543212
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
@@ -4336,6 +4406,16 @@ def test_simple_fft_insert_with_icon(self):
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
+
+ # get the bibdocids of latest file inserted
+ bibdocids = [x.get_bibdocid() for x in BibRecDocs(recid).list_latest_files()]
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(min(bibdocids)))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(max(bibdocids)))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4380,18 +4460,22 @@ def test_multiple_fft_insert(self):
Test University
+ 9876543211
295078
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/9809057.pdf
+ 9876543212
%(sizeofdemobibdata)s
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/demobibdata.xml
+ 9876543213
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
+ 9876543214
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -4410,6 +4494,17 @@ def test_multiple_fft_insert(self):
testrec_expected_urls = []
for files in ('site_logo.gif', 'head.gif', '9809057.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/%(CFG_SITE_RECORD)s/%(recid)s/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'files' : files, 'recid' : recid})
+
+ # replace test buffers with real bibdocid of inserted test files:
+ bibdocids = dict((x.get_full_name(), x.get_bibdocid()) for x in BibRecDocs(recid).list_latest_files())
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(bibdocids.get('9809057.pdf')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(bibdocids.get('demobibdata.xml')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543213',
+ str(bibdocids.get('head.gif')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543214',
+ str(bibdocids.get('site_logo.gif')))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
@@ -4461,6 +4556,7 @@ def test_simple_fft_correct(self):
Test University
+ 987654321
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -4484,6 +4580,13 @@ def test_simple_fft_correct(self):
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
@@ -4560,23 +4663,28 @@ def test_fft_correct_already_exists(self):
Test University
+ 9876543211
35
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif
+ 9876543212
626
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png
+ 9876543213
432
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png
+ 9876543214
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a second description
+ 9876543215
786
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
@@ -4617,6 +4725,19 @@ def test_fft_correct_already_exists(self):
bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid)
+ # replace test buffers with real bibdocid of inserted test file:
+ bibdocids = dict((x.get_full_name(), x.get_bibdocid()) for x in BibRecDocs(recid).list_latest_files())
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(bibdocids.get('line.gif')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(bibdocids.get('line.png')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543213',
+ str(bibdocids.get('rss.png')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543214',
+ str(bibdocids.get('site_logo.gif')))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543215',
+ str(bibdocids.get('site_logo.png')))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
@@ -4659,6 +4780,7 @@ def test_fft_correct_modify_doctype(self):
123456789
SzGeCERN
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a description
@@ -4678,6 +4800,12 @@ def test_fft_correct_modify_doctype(self):
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
+ # get the bibdocid of latest file inserted
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -4729,11 +4857,13 @@ def test_fft_append_already_exists(self):
Test University
+ 9876543211
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a description
+ 9876543212
786
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
@@ -4760,6 +4890,14 @@ def test_fft_append_already_exists(self):
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
self.check_record_consistency(recid)
+ # get the bibdocids of latest file inserted
+ bibdocids = [x.get_bibdocid() for x in BibRecDocs(recid).list_latest_files()]
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(min(bibdocids)))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(max(bibdocids)))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
@@ -4931,6 +5069,7 @@ def test_detailed_fft_correct(self):
123456789
%(siteurl)s/img/head.gif
+ Copyright Holder||1234
site_logo
patata
Next Try
@@ -4949,7 +5088,13 @@ def test_detailed_fft_correct(self):
Test, John
Test University
+
+ 987654321
+ Copyright Holder
+ 1234
+
+ 987654321
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Next Try
@@ -4980,6 +5125,15 @@ def test_detailed_fft_correct(self):
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ # replace test buffers with real bibdocid of inserted test file:
+
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
@@ -5028,6 +5182,7 @@ def test_no_url_fft_correct(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Try
@@ -5056,6 +5211,10 @@ def test_no_url_fft_correct(self):
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
@@ -5096,6 +5255,7 @@ def test_new_icon_fft_append(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
@@ -5123,6 +5283,11 @@ def test_new_icon_fft_append(self):
bibupload.bibupload_records(recs, opt_mode='append')[0]
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
@@ -5140,6 +5305,16 @@ def test_multiple_fft_correct(self):
Test, John
Test University
+
+ General License
+ www.license.url.com
+
+
+ General Copyright Holder
+ jekyll@cds.cern.ch
+ CopyrightMessage
+ 2000
+
%(siteurl)s/img/site_logo.gif
Try
@@ -5163,6 +5338,7 @@ def test_multiple_fft_correct(self):
123456789
%(siteurl)s/img/loading.gif
+ License||www.license.url.com||license body
site_logo
patata
.gif
@@ -5181,7 +5357,24 @@ def test_multiple_fft_correct(self):
Test, John
Test University
+
+ General License
+ www.license.url.com
+
+
+ 987654321
+ License
+ www.license.url.com
+ license body
+
+
+ General Copyright Holder
+ jekyll@cds.cern.ch
+ CopyrightMessage
+ 2000
+
+ 987654321
9427
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
@@ -5208,6 +5401,10 @@ def test_multiple_fft_correct(self):
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
@@ -5269,10 +5466,12 @@ def test_purge_fft_correct(self):
Test University
+ 9876543211
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
+ 9876543212
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
@@ -5303,7 +5502,14 @@ def test_purge_fft_correct(self):
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ bibdocids = [x.get_bibdocid() for x in BibRecDocs(recid).list_latest_files()]
+ # replace test buffers with real bibdocid of inserted test file:
+ testrec_expected_xm = testrec_expected_xm.replace('9876543211',
+ str(min(bibdocids)))
+ testrec_expected_xm = testrec_expected_xm.replace('9876543212',
+ str(max(bibdocids)))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
@@ -5371,6 +5577,7 @@ def test_revert_fft_correct(self):
jekyll@cds.cern.ch
+ 987654321
171
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -5401,6 +5608,13 @@ def test_revert_fft_correct(self):
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
+
# revert test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_revert)
bibupload.bibupload_records(recs, opt_mode='correct')
@@ -5484,6 +5698,7 @@ def test_simple_fft_replace(self):
jekyll@cds.cern.ch
+ 987654321
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
@@ -5514,6 +5729,13 @@ def test_simple_fft_replace(self):
bibupload.bibupload_records(recs, opt_mode='replace')
self.check_record_consistency(recid)
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
+
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
@@ -5585,6 +5807,7 @@ def test_simple_fft_insert_with_modification_time(self):
Test University
+ 987654321
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
@@ -5608,6 +5831,13 @@ def test_simple_fft_insert_with_modification_time(self):
str(recid))
testrec_expected_url2 = testrec_expected_url2.replace('123456789',
str(recid))
+
+ # get the bibdocid of latest file inserted
+ # only ony file is added so we don't have to additionally check
+ # if it's really the last file added
+ bibdocid = BibRecDocs(recid).list_latest_files()[0].get_bibdocid()
+ testrec_expected_xm = testrec_expected_xm.replace('987654321',
+ str(bibdocid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
diff --git a/modules/bibupload/lib/bibupload_revisionverifier_regression_tests.py b/modules/bibupload/lib/bibupload_revisionverifier_regression_tests.py
index 849f94bfd7..e952b12122 100644
--- a/modules/bibupload/lib/bibupload_revisionverifier_regression_tests.py
+++ b/modules/bibupload/lib/bibupload_revisionverifier_regression_tests.py
@@ -1114,7 +1114,7 @@ def test_corrected_record_affected_tags(self):
"""Checks if corrected record has affected fields in hstRECORD table"""
query = "SELECT affected_fields from hstRECORD where id_bibrec=12 ORDER BY job_date DESC"
res = run_sql(query)
- self.assertEqual(res[0][0], "005__%,8564_%,909C0%,909C1%,909C5%,909CO%,909CS%")
+ self.assertEqual(res[0][0], "005__%,540__%,542__%,8564_%,909C0%,909C1%,909C5%,909CO%,909CS%")
def test_append_to_record_affected_tags(self):
diff --git a/modules/miscutil/demo/democfgdata.sql b/modules/miscutil/demo/democfgdata.sql
index 490036c60d..f8904aae57 100755
--- a/modules/miscutil/demo/democfgdata.sql
+++ b/modules/miscutil/demo/democfgdata.sql
@@ -1211,7 +1211,7 @@ INSERT INTO sbmFIELD VALUES ('SRVDEMOPIC',1,2,'DEMOPIC_CONT',' <
INSERT INTO sbmFIELD VALUES ('SBIDEMOTHE',1,1,'DEMOTHE_REP','Submit an ATLANTIS Thesis: (one per line) * Thesis Title:(if any) :* Author of the Thesis: (one per line) * Author of the Thesis:(one per line)
* Abstract:
+
10.5072/invenio.test.1
@@ -78,8 +82,12 @@
if not HAS_SSL:
from warnings import warn
- warn("Module ssl not installed. Please install with e.g. 'pip install ssl'. Required for HTTPS connections to DataCite.", RuntimeWarning)
+ warn("Module ssl not installed. Please install with e.g. "
+ "'pip install ssl'. Required for HTTPS connections to DataCite.",
+ RuntimeWarning)
+import re
+from invenio.xmlDict import XmlDictConfig, ElementTree
# Uncomment to enable debugging of HTTP connection and uncomment line in
# DataCiteRequest.request()
@@ -93,15 +101,18 @@
# OpenSSL 1.0.0 has a reported bug with SSLv3/TLS handshake.
# Python libs affected are httplib2 and urllib2. Eg:
# httplib2.SSLHandshakeError: [Errno 1] _ssl.c:497:
- # error:14077438:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert internal error
- # custom HTTPS opener, banner's oracle 10g server supports SSLv3 only
+ # error:14077438:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert internal
+ # error custom HTTPS opener, banner's oracle 10g server supports SSLv3 only
class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
try:
- sock = socket.create_connection((self.host, self.port), self.timeout)
+ sock = socket.create_connection(
+ (self.host, self.port),
+ self.timeout
+ )
except AttributeError:
# Python 2.4 compatibility (does not deal with IPv6)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -116,9 +127,15 @@ def connect(self):
pass
try:
- self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3)
+ self.sock = ssl.wrap_socket(
+ sock, self.key_file, self.cert_file,
+ ssl_version=ssl.PROTOCOL_TLSv1
+ )
except ssl.SSLError:
- self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
+ self.sock = ssl.wrap_socket(
+ sock, self.key_file, self.cert_file,
+ ssl_version=ssl.PROTOCOL_SSLv23
+ )
class HTTPSHandlerV3(urllib2.HTTPSHandler):
def https_open(self, req):
@@ -176,7 +193,10 @@ class DataCiteRequestError(DataCiteError):
class DataCiteNoContentError(DataCiteRequestError):
- """ DOI is known to MDS, but is not resolvable (might be due to handle's latency) """
+ """
+ DOI is known to MDS, but is not resolvable (might be due to handle's
+ latency)
+ """
pass
@@ -235,7 +255,8 @@ class DataCiteRequest(object):
query string on all requests.
@type default_params: dict
"""
- def __init__(self, base_url=None, username=None, password=None, default_params={}):
+ def __init__(self, base_url=None, username=None, password=None,
+ default_params={}):
self.base_url = base_url
self.username = username
self.password = password
@@ -265,7 +286,8 @@ def request(self, url, method='GET', body=None, params={}, headers={}):
self.data = None
self.code = None
- headers['Authorization'] = 'Basic ' + base64.encodestring(self.username + ':' + self.password)
+ headers['Authorization'] = 'Basic ' + \
+ base64.encodestring(self.username + ':' + self.password)
if headers['Authorization'][-1] == '\n':
headers['Authorization'] = headers['Authorization'][:-1]
@@ -284,7 +306,8 @@ def request(self, url, method='GET', body=None, params={}, headers={}):
# HTTP client requests must end with double newline (not added
# by urllib2)
body += '\r\n\r\n'
- body = body.encode('utf-8')
+ if isinstance(body, unicode):
+ body = body.encode('utf-8')
else:
if params:
url = "%s?%s" % (url, urlencode(params))
@@ -301,10 +324,10 @@ def request(self, url, method='GET', body=None, params={}, headers={}):
res = opener.open(request)
self.code = res.code
self.data = res.read()
- except urllib2.HTTPError, e:
+ except urllib2.HTTPError as e:
self.code = e.code
self.data = e.msg
- except urllib2.URLError, e:
+ except urllib2.URLError as e:
raise HttpError(e)
def get(self, url, params={}, headers={}):
@@ -313,11 +336,13 @@ def get(self, url, params={}, headers={}):
def post(self, url, body=None, params={}, headers={}):
""" Make a POST request """
- return self.request(url, method='POST', body=body, params=params, headers=headers)
+ return self.request(url, method='POST', body=body, params=params,
+ headers=headers)
def delete(self, url, params={}, headers={}):
""" Make a DELETE request """
- return self.request(url, method="DELETE", params=params, headers=headers)
+ return self.request(url, method="DELETE", params=params,
+ headers=headers)
class DataCite(object):
@@ -325,10 +350,11 @@ class DataCite(object):
DataCite API wrapper
"""
- def __init__(self, username=None, password=None, url=None, prefix=None, test_mode=None, api_ver="2"):
+ def __init__(self, username=None, password=None, url=None, prefix=None,
+ test_mode=None, api_ver="2"):
"""
- Initialize DataCite API. In case parameters are not specified via keyword
- arguments, they will be read from the Invenio configuration.
+ Initialize DataCite API. In case parameters are not specified via
+ keyword arguments, they will be read from the Invenio configuration.
@param username: DataCite username (or CFG_DATACITE_USERNAME)
@type username: str
@@ -336,27 +362,37 @@ def __init__(self, username=None, password=None, url=None, prefix=None, test_mod
@param password: DataCite password (or CFG_DATACITE_PASSWORD)
@type password: str
- @param url: DataCite API base URL (or CFG_DATACITE_URL). Defaults to https://mds.datacite.org/.
+ @param url: DataCite API base URL (or CFG_DATACITE_URL). Defaults to
+ https://mds.datacite.org/.
@type url: str
- @param prefix: DOI prefix (or CFG_DATACITE_DOI_PREFIX). Defaults to 10.5072 (DataCite test prefix).
+ @param prefix: DOI prefix (or CFG_DATACITE_DOI_PREFIX). Defaults to
+ 10.5072 (DataCite test prefix).
@type prefix: str
- @param test_mode: Set to True to enable test mode (or CFG_DATACITE_TESTMODE). Defaults to False.
+ @param test_mode: Set to True to enable test mode (or
+ CFG_DATACITE_TESTMODE). Defaults to False.
@type test_mode: boolean
- @param api_ver: DataCite API version. Currently has no effect. Default to 2.
+ @param api_ver: DataCite API version. Currently has no effect.
+ Default to 2.
@type api_ver: str
"""
if not HAS_SSL:
- warn("Module ssl not installed. Please install with e.g. 'pip install ssl'. Required for HTTPS connections to DataCite.")
-
- self.username = username or getattr(config, 'CFG_DATACITE_USERNAME', '')
- self.password = password or getattr(config, 'CFG_DATACITE_PASSWORD', '')
- self.prefix = prefix or getattr(config, 'CFG_DATACITE_DOI_PREFIX', '10.5072')
+ warn("Module ssl not installed. Please install with e.g. "
+ "'pip install ssl'. Required for HTTPS connections to "
+ "DataCite.")
+
+ self.username = username or getattr(config, 'CFG_DATACITE_USERNAME',
+ '')
+ self.password = password or getattr(config, 'CFG_DATACITE_PASSWORD',
+ '')
+ self.prefix = prefix or getattr(config, 'CFG_DATACITE_DOI_PREFIX',
+ '10.5072')
self.api_ver = api_ver # Currently not used
- self.api_url = url or getattr(config, 'CFG_DATACITE_URL', 'https://mds.datacite.org/')
+ self.api_url = url or getattr(config, 'CFG_DATACITE_URL',
+ 'https://mds.datacite.org/')
if self.api_url[-1] != '/':
self.api_url = self.api_url + "/"
@@ -535,3 +571,86 @@ def media_post(self, doi, media):
return r.data
else:
raise DataCiteError.factory(r.code)
+
+
+class DataciteMetadata(object):
+
+ def __init__(self, doi):
+
+ self.url = "http://data.datacite.org/application/x-datacite+xml/"
+ self.error = False
+ try:
+ data = urllib2.urlopen(self.url + doi).read()
+ except urllib2.HTTPError:
+ self.error = True
+
+ if not self.error:
+ # Clean the xml for parsing
+ data = re.sub('<\?xml.*\?>', '', data, count=1)
+
+ # Remove the resource tags
+ data = re.sub('', '', data)
+ self.data = '' + \
+ data[0:len(data) - 11] + ' '
+ self.root = ElementTree.XML(self.data)
+ self.xml = XmlDictConfig(self.root)
+
+ def get_creators(self, attribute='creatorName'):
+ if 'creators' in self.xml:
+ if isinstance(self.xml['creators']['creator'], list):
+ return [c[attribute] for c in self.xml['creators']['creator']]
+ else:
+ return self.xml['creators']['creator'][attribute]
+
+ return None
+
+ def get_titles(self):
+ if 'titles' in self.xml:
+ return self.xml['titles']['title']
+ return None
+
+ def get_publisher(self):
+ if 'publisher' in self.xml:
+ return self.xml['publisher']
+ return None
+
+ def get_dates(self):
+ if 'dates' in self.xml:
+ if isinstance(self.xml['dates']['date'], dict):
+ return self.xml['dates']['date'].values()[0]
+ return self.xml['dates']['date']
+ return None
+
+ def get_publication_year(self):
+ if 'publicationYear' in self.xml:
+ return self.xml['publicationYear']
+ return None
+
+ def get_language(self):
+ if 'language' in self.xml:
+ return self.xml['language']
+ return None
+
+ def get_related_identifiers(self):
+ pass
+
+ def get_description(self, description_type='Abstract'):
+ if 'descriptions' in self.xml:
+ if isinstance(self.xml['descriptions']['description'], list):
+ for description in self.xml['descriptions']['description']:
+ if description_type in description:
+ return description[description_type]
+ elif isinstance(self.xml['descriptions']['description'], dict):
+ description = self.xml['descriptions']['description']
+ if description_type in description:
+ return description[description_type]
+ elif len(description) == 1:
+ # return the only description
+ return description.values()[0]
+
+ return None
+
+ def get_rights(self):
+ if 'titles' in self.xml:
+ return self.xml['rights']
+ return None
diff --git a/modules/miscutil/lib/elasticsearch_logging.py b/modules/miscutil/lib/elasticsearch_logging.py
new file mode 100644
index 0000000000..d2308d6816
--- /dev/null
+++ b/modules/miscutil/lib/elasticsearch_logging.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014, 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.config import (
+ CFG_ELASTICSEARCH_FALLBACK_DIRECTORY,
+ CFG_ELASTICSEARCH_FLUSH_INTERVAL,
+ CFG_ELASTICSEARCH_HOSTS,
+ CFG_ELASTICSEARCH_INDEX_PREFIX,
+ CFG_ELASTICSEARCH_LOGGING,
+ CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH,
+ CFG_ELASTICSEARCH_SUFFIX_FORMAT,
+)
+
+if CFG_ELASTICSEARCH_LOGGING:
+ import logging
+ import lumberjack
+ import os
+ import socket
+ import sys
+
+def initialise_lumberjack():
+ if not CFG_ELASTICSEARCH_LOGGING:
+ return None
+ config = lumberjack.get_default_config()
+ config['index_prefix'] = CFG_ELASTICSEARCH_INDEX_PREFIX
+
+ if CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH == -1:
+ config['max_queue_length'] = None
+ else:
+ config['max_queue_length'] = CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH
+
+ if CFG_ELASTICSEARCH_FLUSH_INTERVAL == -1:
+ config['interval'] = None
+ else:
+ config['interval'] = CFG_ELASTICSEARCH_FLUSH_INTERVAL
+
+ # Check if lumberjack directory exists
+ _lumberjack_exists(CFG_ELASTICSEARCH_FALLBACK_DIRECTORY)
+
+ # Get hostname from socket (strip cern.ch)
+ fallback_file_name = socket.getfqdn().replace('.cern.ch', '')
+
+ fallback_file_path = os.path.join(
+ CFG_ELASTICSEARCH_FALLBACK_DIRECTORY,
+ fallback_file_name
+ )
+ config['fallback_log_file'] = fallback_file_path
+
+ lj = lumberjack.Lumberjack(
+ hosts=CFG_ELASTICSEARCH_HOSTS,
+ config=config)
+
+ handler = lj.get_handler(suffix_format=CFG_ELASTICSEARCH_SUFFIX_FORMAT)
+ logging.getLogger('events').addHandler(handler)
+ logging.getLogger('events').setLevel(logging.INFO)
+
+ logging.getLogger('lumberjack').addHandler(
+ logging.StreamHandler(sys.stderr))
+ logging.getLogger('lumberjack').setLevel(logging.ERROR)
+
+ return lj
+
+
+def _lumberjack_exists(path):
+ """Check if lumberjack directory exists.
+
+ :param str path: the path to lumberjack directory
+ """
+ try:
+ os.makedirs(path)
+ except OSError:
+ if not os.path.isdir(path):
+ raise
+
+LUMBERJACK = initialise_lumberjack()
+
+
+def register_schema(*args, **kwargs):
+ if not CFG_ELASTICSEARCH_LOGGING:
+ return None
+ return LUMBERJACK.register_schema(*args, **kwargs)
diff --git a/modules/miscutil/lib/errorlib.py b/modules/miscutil/lib/errorlib.py
index d2ceaaaf78..6b2114b8de 100644
--- a/modules/miscutil/lib/errorlib.py
+++ b/modules/miscutil/lib/errorlib.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2014 CERN.
+# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -29,6 +29,7 @@
import re
import inspect
import json
+import logging
from cStringIO import StringIO
@@ -426,7 +427,9 @@ def default(self, obj):
client.extra_context(user_info)
filename = _get_filename_and_line(sys.exc_info())[0]
client.tags_context({'filename': filename, 'version': CFG_VERSION})
- client.captureException()
+ client.captureException(
+ level=_guess_exception_level(sys.exc_info())
+ )
except Exception:
# Exception management of exception management
try:
@@ -607,15 +610,37 @@ def send_error_report_to_admin(header, url, time_msg,
from invenio.mailutils import send_email
send_email(from_addr, to_addr, subject="Error notification", content=body)
+
+def _guess_exception_level(exc_info):
+ """Set the logging level depending on the exception name."""
+ try:
+ if hasattr(exc_info[1], 'level'):
+ return exc_info[1].level
+
+ if 'warning' in exc_info[0].__name__.lower():
+ return logging.WARN
+ if 'info' in exc_info[0].__name__.lower():
+ return logging.INFO
+ if 'error' in exc_info[0].__name__.lower():
+ return logging.ERROR
+ except AttributeError:
+ pass
+
+ return logging.ERROR
+
+
def _get_filename_and_line(exc_info):
"""
Return the filename, the line and the function_name where the exception happened.
"""
tb = exc_info[2]
- exception_info = traceback.extract_tb(tb)[-1]
- filename = os.path.basename(exception_info[0])
- line_no = exception_info[1]
- function_name = exception_info[2]
+ try:
+ exception_info = traceback.extract_tb(tb)[-1]
+ filename = os.path.basename(exception_info[0])
+ line_no = exception_info[1]
+ function_name = exception_info[2]
+ except IndexError:
+ return '', '', ''
return filename, line_no, function_name
def _truncate_dynamic_string(val, maxlength=500):
diff --git a/modules/miscutil/lib/htmlutils.py b/modules/miscutil/lib/htmlutils.py
index fbeeb3366b..b1b4c26e8d 100644
--- a/modules/miscutil/lib/htmlutils.py
+++ b/modules/miscutil/lib/htmlutils.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
-#
+
# This file is part of Invenio.
-# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2013 CERN.
+# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
"""HTML utilities."""
__revision__ = "$Id$"
@@ -491,6 +492,10 @@ def get_html_text_editor(name, id=None, content='', textual_content=None, width=
like replace CRLF with
+ */
+ if ( e.data.keyCode == 13 ) {
+
+ // The following will break all blockquotes, one Enter at a time
+ var selection = oEditor.getSelection();
+ var element = selection.getStartElement();
+ var parent = element.getParent();
+ var range_split_block = true;
+
+ if ( element.is("blockquote") && parent.is("body") ) {
+ if ( element.getText().trim() == "" ) {
+ element.remove();
+ range_split_block = false;
+ // Adding an empty paragraph seems to make it smoother
+ CKEDITOR.instances.msg.insertHtml("
");
+ }
+ }
+
+ if ( range_split_block == true ) {
+ var ranges = selection.getRanges(true);
+ for ( var i = ranges.length - 1 ; i > 0 ; i-- ) {
+ ranges[i].deleteContents();
+ }
+ var range = ranges[0];
+ range.splitBlock("blockquote");
+ if ( ! ( element.is("p") && parent.is("body") ) ) {
+ // Adding an empty paragraph seems to make it smoother
+ CKEDITOR.instances.msg.insertHtml("
");
+ }
+ }
+
+ }
+
+ });
+
})
//]]>
diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py
index 45132b5f0d..6747a0d1d6 100644
--- a/modules/miscutil/lib/inveniocfg.py
+++ b/modules/miscutil/lib/inveniocfg.py
@@ -189,7 +189,9 @@ def convert_conf_option(option_name, option_value):
'CFG_REDIS_HOSTS',
'CFG_BIBSCHED_INCOMPATIBLE_TASKS',
'CFG_ICON_CREATION_FORMAT_MAPPINGS',
- 'CFG_BIBEDIT_AUTOCOMPLETE']:
+ 'CFG_BIBEDIT_AUTOCOMPLETE',
+ 'CFG_ELASTICSEARCH_HOSTS',
+ 'CFG_ELASTICSEARCH_BOT_AGENT_STRINGS']:
try:
option_value = option_value[1:-1]
if option_name == "CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE" and option_value.strip().startswith("{"):
@@ -245,7 +247,8 @@ def convert_conf_option(option_name, option_value):
'CFG_OAUTH2_PROVIDERS',
'CFG_BIBFORMAT_CACHED_FORMATS',
'CFG_BIBEDIT_ADD_TICKET_RT_QUEUES',
- 'CFG_BIBAUTHORID_ENABLED_REMOTE_LOGIN_SYSTEMS',]:
+ 'CFG_BIBAUTHORID_ENABLED_REMOTE_LOGIN_SYSTEMS',
+ 'PIDSTORE_OBJECT_TYPES', ]:
out = "["
for elem in option_value[1:-1].split(","):
if elem:
diff --git a/modules/miscutil/lib/pid_provider.py b/modules/miscutil/lib/pid_provider.py
new file mode 100644
index 0000000000..fa6c70fa5c
--- /dev/null
+++ b/modules/miscutil/lib/pid_provider.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.containerutils import LazyDict
+
+
+class PidProvider(object):
+ """
+ Abstract class for persistent identifier provider classes.
+
+ Subclasses must implement register, update, delete and is_provider_for_pid
+ methods and register itself:
+
+ class MyProvider(PidProvider):
+ pid_type = "mypid"
+
+ def reserve(self, pid, *args, **kwargs):
+ return True
+
+ def register(self, pid, *args, **kwargs):
+ return True
+
+ def update(self, pid, *args, **kwargs):
+ return True
+
+ def delete(self, pid, *args, **kwargs):
+ try:
+ ...
+ except Exception as e:
+ pid.log("DELETE","Deletion failed")
+ return False
+ else:
+ pid.log("DELETE","Successfully deleted")
+ return True
+
+ def is_provider_for_pid(self, pid_str):
+ pass
+
+ PidProvider.register_provider(MyProvider)
+
+
+ The provider is responsible for handling of errors, as well as logging of
+ actions happening to the pid. See example above as well as the
+ DataCitePidProvider.
+
+ Each method takes variable number of argument and keywords arguments. This
+ can be used to pass additional information to the provider when registering
+ a persistent identifier. E.g. a DOI requires URL and metadata to be able
+ to register the DOI.
+ """
+
+ def __load_providers():
+ from invenio.pid_store import _PID_PROVIDERS
+ registry = dict()
+ for provider in _PID_PROVIDERS.values():
+ if not issubclass(provider, PidProvider):
+ raise TypeError("Argument not an instance of PidProvider.")
+ pid_type = getattr(provider, 'pid_type', None)
+ if pid_type is None:
+ raise AttributeError(
+ "Provider must specify class variable pid_type.")
+ pid_type = pid_type.lower()
+ if pid_type not in registry:
+ registry[pid_type] = []
+
+ # Prevent double registration
+ if provider not in registry[pid_type]:
+ registry[pid_type].append(provider)
+ return registry
+
+ registry = LazyDict(__load_providers)
+ """ Registry of possible providers """
+
+ pid_type = None
+ """
+ Must be overwritten in subcleass and specified as a string (max len 6)
+ """
+
+ @staticmethod
+ def create(pid_type, pid_str, pid_provider, *args, **kwargs):
+ """
+ Create a new instance of a PidProvider for the
+ given type and pid.
+ """
+ providers = PidProvider.registry.get(pid_type.lower(), None)
+ for p in providers:
+ if p.is_provider_for_pid(pid_str):
+ return p(*args, **kwargs)
+ return None
+
+ #
+ # API methods which must be implemented by each provider.
+ #
+ def reserve(self, pid, *args, **kwargs):
+ """
+ Reserve a new persistent identifier
+
+ This might or might not be useful depending on the service of the
+ provider.
+ """
+ raise NotImplementedError
+
+ def register(self, pid, *args, **kwargs):
+ """ Register a new persistent identifier """
+ raise NotImplementedError
+
+ def update(self, pid, *args, **kwargs):
+ """ Update information about a persistent identifier """
+ raise NotImplementedError
+
+ def delete(self, pid, *args, **kwargs):
+ """ Delete a persistent identifier """
+ raise NotImplementedError
+
+ def sync_status(self, pid, *args, **kwargs):
+ """
+ Synchronize persistent identifier status with remote service provider.
+ """
+ return True
+
+ @classmethod
+ def is_provider_for_pid(cls, pid_str):
+ raise NotImplementedError
+
+ #
+ # API methods which might need to be implemented depending on each provider.
+ #
+ def create_new_pid(self, pid_value):
+ """ Some PidProvider might have the ability to create new values """
+ return pid_value
+
+class LocalPidProvider(PidProvider):
+ """
+ Abstract class for local persistent identifier provides (i.e locally
+ unmanaged DOIs).
+ """
+ def reserve(self, pid, *args, **kwargs):
+ pid.log("RESERVE", "Successfully reserved locally")
+ return True
+
+ def register(self, pid, *args, **kwargs):
+ pid.log("REGISTER", "Successfully registered in locally")
+ return True
+
+ def update(self, pid, *args, **kwargs):
+ # No logging necessary as status of PID is not changing
+ return True
+
+ def delete(self, pid, *args, **kwargs):
+ """ Delete a registered DOI """
+ pid.log("DELETE", "Successfully deleted locally")
+ return True
diff --git a/modules/miscutil/lib/pid_providers/Makefile.am b/modules/miscutil/lib/pid_providers/Makefile.am
new file mode 100644
index 0000000000..551fc6458d
--- /dev/null
+++ b/modules/miscutil/lib/pid_providers/Makefile.am
@@ -0,0 +1,24 @@
+## This file is part of Invenio.
+## Copyright (C) 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+pylibdir = $(libdir)/python/invenio/pid_providers
+
+pylib_DATA = *.py
+
+EXTRA_DIST = $(pylib_DATA)
+
+CLEANFILES = *~ *.tmp *.pyc
diff --git a/modules/miscutil/lib/pid_providers/__init__.py b/modules/miscutil/lib/pid_providers/__init__.py
new file mode 100644
index 0000000000..0eab0f8880
--- /dev/null
+++ b/modules/miscutil/lib/pid_providers/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
diff --git a/modules/miscutil/lib/pid_providers/datacite.py b/modules/miscutil/lib/pid_providers/datacite.py
new file mode 100644
index 0000000000..4488c3671b
--- /dev/null
+++ b/modules/miscutil/lib/pid_providers/datacite.py
@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014, 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ DataCite PID provider.
+"""
+
+from invenio import config
+from invenio.dataciteutils import DataCite as DataCiteUtil, HttpError, \
+ DataCiteError, DataCiteGoneError, DataCiteNoContentError, \
+ DataCiteNotFoundError
+
+from invenio.pid_provider import PidProvider
+from invenio.pid_store import PIDSTORE_STATUS_NEW, \
+ PIDSTORE_STATUS_REGISTERED, \
+ PIDSTORE_STATUS_DELETED, \
+ PIDSTORE_STATUS_RESERVED
+
+
+class DataCite(PidProvider):
+ """
+ DOI provider using DataCite API.
+ """
+ pid_type = 'doi'
+
+ def __init__(self):
+ self.api = DataCiteUtil()
+
+ def _get_url(self, kwargs):
+ try:
+ return kwargs['url']
+ except KeyError:
+ raise Exception("url keyword argument must be specified.")
+
+ def _get_doc(self, kwargs):
+ try:
+ return kwargs['doc']
+ except KeyError:
+ raise Exception("doc keyword argument must be specified.")
+
+ def reserve(self, pid, *args, **kwargs):
+ """ Reserve a DOI (amounts to upload metadata, but not to mint) """
+ # Only registered PIDs can be updated.
+ doc = self._get_doc(kwargs)
+
+ try:
+ self.api.metadata_post(doc)
+ except DataCiteError as e:
+ pid.log("RESERVE", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("RESERVE", "Failed with HttpError - %s" % unicode(e))
+ return False
+ else:
+ pid.log("RESERVE", "Successfully reserved in DataCite")
+ return True
+
+ def register(self, pid, *args, **kwargs):
+ """ Register a DOI via the DataCite API """
+ url = self._get_url(kwargs)
+ doc = self._get_doc(kwargs)
+
+ try:
+ # Set metadata for DOI
+ self.api.metadata_post(doc)
+ # Mint DOI
+ self.api.doi_post(pid.pid_value, url)
+ except DataCiteError as e:
+ pid.log("REGISTER", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("REGISTER", "Failed with HttpError - %s" % unicode(e))
+ return False
+ else:
+ pid.log("REGISTER", "Successfully registered in DataCite")
+ return True
+
+ def update(self, pid, *args, **kwargs):
+ """
+ Update metadata associated with a DOI.
+
+ This can be called before/after a DOI is registered
+
+ """
+ url = self._get_url(kwargs)
+ doc = self._get_doc(kwargs)
+
+ if pid.is_deleted():
+ pid.log("UPDATE", "Reactivate in DataCite")
+
+ try:
+ # Set metadata
+ self.api.metadata_post(doc)
+ self.api.doi_post(pid.pid_value, url)
+ except DataCiteError as e:
+ pid.log("UPDATE", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("UPDATE", "Failed with HttpError - %s" % unicode(e))
+ return False
+ else:
+ if pid.is_deleted():
+ pid.log(
+ "UPDATE",
+ "Successfully updated and possibly registered in DataCite"
+ )
+ else:
+ pid.log("UPDATE", "Successfully updated in DataCite")
+ return True
+
+ def delete(self, pid, *args, **kwargs):
+ """ Delete a registered DOI """
+ try:
+ self.api.metadata_delete(pid.pid_value)
+ except DataCiteError as e:
+ pid.log("DELETE", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("DELETE", "Failed with HttpError - %s" % unicode(e))
+ return False
+ else:
+ pid.log("DELETE", "Successfully deleted in DataCite")
+ return True
+
+ def sync_status(self, pid, *args, **kwargs):
+ """ Synchronize DOI status DataCite MDS """
+ status = None
+
+ try:
+ self.api.doi_get(pid.pid_value)
+ status = PIDSTORE_STATUS_REGISTERED
+ except DataCiteGoneError:
+ status = PIDSTORE_STATUS_DELETED
+ except DataCiteNoContentError:
+ status = PIDSTORE_STATUS_REGISTERED
+ except DataCiteNotFoundError:
+ pass
+ except DataCiteError as e:
+ pid.log("SYNC", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("SYNC", "Failed with HttpError - %s" % unicode(e))
+ return False
+
+ if status is None:
+ try:
+ self.api.metadata_get(pid.pid_value)
+ status = PIDSTORE_STATUS_RESERVED
+ except DataCiteGoneError:
+ status = PIDSTORE_STATUS_DELETED
+ except DataCiteNoContentError:
+ status = PIDSTORE_STATUS_REGISTERED
+ except DataCiteNotFoundError:
+ pass
+ except DataCiteError as e:
+ pid.log("SYNC", "Failed with %s" % e.__class__.__name__)
+ return False
+ except HttpError as e:
+ pid.log("SYNC", "Failed with HttpError - %s" % unicode(e))
+ return False
+
+ if status is None:
+ status = PIDSTORE_STATUS_NEW
+
+ if pid.status != status:
+ pid.log(
+ "SYNC", "Fixed status from %s to %s." % (pid.status, status)
+ )
+ pid.status = status
+
+ return True
+
+ @classmethod
+ def is_provider_for_pid(cls, pid_str):
+ """
+ Check if DataCite is the provider for this DOI
+
+ Note: If you e.g. changed DataCite account and received a new prefix,
+ then this provider can only update and register DOIs for the new
+ prefix.
+ """
+ CFG_DATACITE_DOI_PREFIX = getattr(config,
+ 'CFG_DATACITE_DOI_PREFIX',
+ '10.0572')
+ return pid_str.startswith("%s/" % CFG_DATACITE_DOI_PREFIX)
+
+provider = DataCite
diff --git a/modules/miscutil/lib/pid_providers/local_doi.py b/modules/miscutil/lib/pid_providers/local_doi.py
new file mode 100644
index 0000000000..adc57490a8
--- /dev/null
+++ b/modules/miscutil/lib/pid_providers/local_doi.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+LocalDOI provider.
+"""
+
+from invenio import config
+
+from invenio.pid_provider import LocalPidProvider
+
+
+class LocalDOI(LocalPidProvider):
+ """
+ Provider for locally unmanaged DOIs.
+ """
+ pid_type = 'doi'
+
+ @classmethod
+ def is_provider_for_pid(cls, pid_str):
+ """
+ Check if DOI is not the local datacite managed one.
+ """
+ CFG_DATACITE_DOI_PREFIX = getattr(config,
+ 'CFG_DATACITE_DOI_PREFIX',
+ '10.5072')
+ return pid_str.startswith("%s/" % CFG_DATACITE_DOI_PREFIX)
+
+provider = LocalDOI
diff --git a/modules/miscutil/lib/pid_store.py b/modules/miscutil/lib/pid_store.py
new file mode 100644
index 0000000000..13f724ad9f
--- /dev/null
+++ b/modules/miscutil/lib/pid_store.py
@@ -0,0 +1,432 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2015 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""PersistentIdentifier store and registration.
+
+Usage example for registering new identifiers::
+
+ from flask import url_for
+ from invenio.pid_store import PersistentIdentifier
+
+ # Reserve a new DOI internally first
+ pid = PersistentIdentifier.create('doi','10.0572/1234')
+
+ # Get an already reserved DOI
+ pid = PersistentIdentifier.get('doi', '10.0572/1234')
+
+ # Assign it to a record.
+ pid.assign('rec', 1234)
+
+ url = url_for("record.metadata", recid=1234, _external=True)
+ doc = " 0:
+ # check if user can view record
+ user_info = collect_user_info(user_id)
+ if check_user_can_view_record(user_info, recid)[0] > 0:
+ # Not authorized
+ continue
+ else:
+ # check if record is public
+ if not record_public_p(recid):
+ continue
+
+ # get record information
+ record = get_record(recid)
+ title = record.get('title.title')
+
+ # Check for no title or similar title
+ if not title or title in check_same_title:
+ continue
+ else:
+ check_same_title.append(title)
+
+ rec_authors = filter(None, record.get('authors.full_name', [])) \
+ or filter(None, record.get('corporate_name.name', []))
+ authors = "; ".join(rec_authors)
+ except (KeyError, TypeError, ValueError, AttributeError):
+ continue
+
+ record_url = "{0}/{1}/{2}".format(CFG_BASE_URL, CFG_SITE_RECORD,
+ str(recid))
+ url = get_url_customevent(record_url,
+ "recommended_record",
+ [str(recid_source), str(recid),
+ str(rec_count), str(user_id),
+ str(recommender_version)])
+ suggestions.append({
+ 'number': rec_count,
+ 'record_url': url,
+ 'record_title': title.strip(),
+ 'record_authors': authors.strip(),
+ })
+ if rec_count >= maximum:
+ break
+ rec_count += 1
+
+ return suggestions
diff --git a/modules/miscutil/lib/recommender_initializer.py b/modules/miscutil/lib/recommender_initializer.py
new file mode 100644
index 0000000000..c0e7f119a4
--- /dev/null
+++ b/modules/miscutil/lib/recommender_initializer.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2016 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Initializes the Redis connection."""
+
+from invenio.config import CFG_RECOMMENDER_REDIS
+
+
+_RECOMMENDATION_REDIS = None
+
+
+def get_redis_connection():
+ """
+ Stores a persistent Redis connection.
+
+ @return: Redis connection object.
+ """
+ global _RECOMMENDATION_REDIS
+
+ if CFG_RECOMMENDER_REDIS == "":
+ # Recommender is not configured.
+ return None
+
+ if _RECOMMENDATION_REDIS is None:
+ import redis
+ _RECOMMENDATION_REDIS = redis.StrictRedis(host=CFG_RECOMMENDER_REDIS,
+ port=6379,
+ db=0)
+ return _RECOMMENDATION_REDIS
diff --git a/modules/miscutil/lib/solrutils_bibrank_indexer.py b/modules/miscutil/lib/solrutils_bibrank_indexer.py
index fd0f83da19..834092f08b 100644
--- a/modules/miscutil/lib/solrutils_bibrank_indexer.py
+++ b/modules/miscutil/lib/solrutils_bibrank_indexer.py
@@ -22,17 +22,35 @@
"""
+import os
+import urllib2
+import re
import time
-from invenio.config import CFG_SOLR_URL
+
+from invenio.config import (
+ CFG_SOLR_URL,
+ CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY,
+ CFG_BIBINDEX_SPLASH_PAGES
+)
from invenio.bibtask import write_message, task_get_option, task_update_progress, \
task_sleep_now_if_required
+from invenio.htmlutils import get_links_in_html_page
+from invenio.websubmit_file_converter import convert_file
from invenio.dbquery import run_sql
-from invenio.search_engine import record_exists
-from invenio.bibdocfile import BibRecDocs
+from invenio.search_engine import record_exists, get_field_tags
+from invenio.search_engine_utils import get_fieldvalues
+from invenio.bibdocfile import BibRecDocs, bibdocfile_url_p, download_url
from invenio.solrutils_bibindex_indexer import replace_invalid_solr_characters
from invenio.bibindex_engine import create_range_list
from invenio.errorlib import register_exception
from invenio.bibrank_bridge_utils import get_tags, get_field_content_in_utf8
+from invenio.bibtask import write_message
+
+
+SOLR_CONNECTION = None
+
+
+SOLR_CONNECTION = None
if CFG_SOLR_URL:
@@ -103,16 +121,11 @@ def solr_add_range(lower_recid, upper_recid, tags_to_index, next_commit_counter)
"""
for recid in range(lower_recid, upper_recid + 1):
if record_exists(recid):
- abstract = get_field_content_in_utf8(recid, 'abstract', tags_to_index)
- author = get_field_content_in_utf8(recid, 'author', tags_to_index)
- keyword = get_field_content_in_utf8(recid, 'keyword', tags_to_index)
- title = get_field_content_in_utf8(recid, 'title', tags_to_index)
- try:
- bibrecdocs = BibRecDocs(recid)
- fulltext = unicode(bibrecdocs.get_text(), 'utf-8')
- except:
- fulltext = ''
-
+ abstract = get_field_content_in_utf8(recid, 'abstract', tags_to_index)
+ author = get_field_content_in_utf8(recid, 'author', tags_to_index)
+ keyword = get_field_content_in_utf8(recid, 'keyword', tags_to_index)
+ title = get_field_content_in_utf8(recid, 'title', tags_to_index)
+ fulltext = _get_fulltext(recid)
solr_add(recid, abstract, author, fulltext, keyword, title)
next_commit_counter = solr_commit_if_necessary(next_commit_counter,recid=recid)
@@ -182,3 +195,48 @@ def word_index(run): # pylint: disable=W0613
write_message("No new records. Solr index is up to date")
write_message("Solr ranking indexer completed")
+
+
+def _get_fulltext(recid):
+
+ words = []
+ try:
+ bibrecdocs = BibRecDocs(recid)
+ words.append(unicode(bibrecdocs.get_text(), 'utf-8'))
+ except Exception, e:
+ pass
+
+ if CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY:
+ write_message("... %s is external URL but indexing only local files" % url, verbose=2)
+ return ' '.join(words)
+
+ urls_from_record = [url for tag in get_field_tags('fulltext')
+ for url in get_fieldvalues(recid, tag)
+ if not bibdocfile_url_p(url)]
+ urls_to_index = set()
+ for url_direct_or_indirect in urls_from_record:
+ for splash_re, url_re in CFG_BIBINDEX_SPLASH_PAGES.iteritems():
+ if re.match(splash_re, url_direct_or_indirect):
+ if url_re is None:
+ write_message("... %s is file to index (%s)" % (url_direct_or_indirect, splash_re), verbose=2)
+ urls_to_index.add(url_direct_or_indirect)
+ continue
+ write_message("... %s is a splash page (%s)" % (url_direct_or_indirect, splash_re), verbose=2)
+ html = urllib2.urlopen(url_direct_or_indirect).read()
+ urls = get_links_in_html_page(html)
+ write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3)
+ for url in urls:
+ if re.match(url_re, url):
+ write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2)
+ urls_to_index.add(url)
+ if urls_to_index:
+ write_message("... will extract words from %s:%s" % (recid, ', '.join(urls_to_index)), verbose=2)
+ for url in urls_to_index:
+ tmpdoc = download_url(url)
+ try:
+ tmptext = convert_file(tmpdoc, output_format='.txt')
+ words.append(open(tmptext).read())
+ os.remove(tmptext)
+ finally:
+ os.remove(tmpdoc)
+ return ' '.join(words)
diff --git a/modules/miscutil/lib/solrutils_bibrank_searcher.py b/modules/miscutil/lib/solrutils_bibrank_searcher.py
index 28aa607467..0ac8ea2bab 100644
--- a/modules/miscutil/lib/solrutils_bibrank_searcher.py
+++ b/modules/miscutil/lib/solrutils_bibrank_searcher.py
@@ -25,6 +25,7 @@
import itertools
+from math import ceil
from invenio.config import CFG_SOLR_URL
from invenio.intbitset import intbitset
from invenio.errorlib import register_exception
@@ -106,7 +107,11 @@ def get_normalized_ranking_scores(response, hitset_filter = None, recids = []):
if (not hitset_filter and hitset_filter != []) or recid in hitset_filter or recid in recids:
normalised_score = 0
if max_score > 0:
- normalised_score = int(100.0 / max_score * float(hit['score']))
+ # Ceil score, in particular beneficial for scores in (0,1) and (99,100) to take 1 and 100
+ normalised_score = int(ceil(100.0 / max_score * float(hit['score'])))
+ # Correct possible rounding error
+ if normalised_score > 100:
+ normalised_score = 100
ranked_result.append((recid, normalised_score))
matched_recs.add(recid)
@@ -115,7 +120,7 @@ def get_normalized_ranking_scores(response, hitset_filter = None, recids = []):
return (ranked_result, matched_recs)
-def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranked_result_amount):
+def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranked_result_amount, kwargs={}):
"""
Ranking a records containing specified words and returns a sorted list.
input:
@@ -138,6 +143,9 @@ def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranke
if pattern:
pattern = " ".join(map(str, pattern))
from invenio.search_engine import create_basic_search_units
+ # Rank global index for fulltext by default in add to search
+ if kwargs.get('aas', 0) == 2 and explicit_field == 'fulltext':
+ explicit_field = ''
search_units = create_basic_search_units(None, pattern, explicit_field)
else:
return (None, "Records not ranked. The query is not detailed enough, or not enough records found, for ranking to be possible.", "", voutput)
diff --git a/modules/miscutil/lib/solrutils_regression_tests.py b/modules/miscutil/lib/solrutils_regression_tests.py
index 5dddd58d3e..fe471985f4 100644
--- a/modules/miscutil/lib/solrutils_regression_tests.py
+++ b/modules/miscutil/lib/solrutils_regression_tests.py
@@ -24,30 +24,50 @@
from invenio import intbitset
from invenio.solrutils_bibindex_searcher import solr_get_bitset
from invenio.solrutils_bibrank_searcher import solr_get_ranked, solr_get_similar_ranked
+from invenio.solrutils_bibrank_indexer import solr_add, SOLR_CONNECTION
from invenio.search_engine import get_collection_reclist
from invenio.bibrank_bridge_utils import get_external_word_similarity_ranker, \
get_logical_fields, \
get_tags, \
get_field_content_in_utf8
+
ROWS = 100
HITSETS = {
'Willnotfind': intbitset.intbitset([]),
- 'higgs': intbitset.intbitset([47, 48, 51, 52, 55, 56, 58, 68, 79, 85, 89, 96]),
- 'of': intbitset.intbitset([8, 10, 11, 12, 15, 43, 44, 45, 46, 47, 48, 49, 50, 51,
- 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 64, 68, 74,
- 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,
- 91, 92, 93, 94, 95, 96, 97]),
- '"higgs boson"': intbitset.intbitset([55, 56]),
+ 'of': intbitset.intbitset([1, 2, 7, 8, 10]),
+ '"of the"': intbitset.intbitset([1, 2, 7, 8, 10])
}
-def get_topN(n, data):
- res = dict()
- for key, value in data.iteritems():
- res[key] = value[-n:]
- return res
+
+RECORDS = xrange(1, 11)
+
+
+TAGS = {'abstract': ['520__%'],
+ 'author': ['100__a', '700__a'],
+ 'keyword': ['6531_a'],
+ 'title': ['245__%', '246__%']}
+
+
+def init_Solr():
+ _delete_all()
+ _index_records()
+ SOLR_CONNECTION.commit()
+
+
+def _delete_all():
+ SOLR_CONNECTION.delete_query('*:*')
+
+
+def _index_records():
+ for recid in RECORDS:
+ fulltext = abstract = get_field_content_in_utf8(recid, 'abstract', TAGS)
+ author = get_field_content_in_utf8(recid, 'author', TAGS)
+ keyword = get_field_content_in_utf8(recid, 'keyword', TAGS)
+ title = get_field_content_in_utf8(recid, 'title', TAGS)
+ solr_add(recid, abstract, author, fulltext, keyword, title)
class TestSolrSearch(InvenioTestCase):
@@ -55,23 +75,19 @@ class TestSolrSearch(InvenioTestCase):
make install-solrutils
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
- AND EITHER
- Solr index built: ./bibindex -w fulltext for all records
- OR
- WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- and ./bibrank -w wrd for all records
"""
-
- def _get_result(self, query, index='fulltext'):
+ def get_result(self, query, index='fulltext'):
return solr_get_bitset(index, query)
+ def setUp(self):
+ init_Solr()
+
@nottest
def test_get_bitset(self):
"""solrutils - search results"""
- self.assertEqual(HITSETS['Willnotfind'], self._get_result('Willnotfind'))
- self.assertEqual(HITSETS['higgs'], self._get_result('higgs'))
- self.assertEqual(HITSETS['of'], self._get_result('of'))
- self.assertEqual(HITSETS['"higgs boson"'], self._get_result('"higgs boson"'))
+ self.assertEqual(self.get_result('Willnotfind'), HITSETS['Willnotfind'])
+ self.assertEqual(self.get_result('of'), HITSETS['of'])
+ self.assertEqual(self.get_result('"of the"'), HITSETS['"of the"'])
class TestSolrRanking(InvenioTestCase):
@@ -79,91 +95,25 @@ class TestSolrRanking(InvenioTestCase):
make install-solrutils
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
- AND EITHER
- Solr index built: ./bibindex -w fulltext for all records
- OR
- WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- and ./bibrank -w wrd for all records
"""
-
- def _get_ranked_result_sequence(self, query, index='fulltext', rows=ROWS, hitset=None):
- if hitset is None:
- hitset=HITSETS[query]
- ranked_result = solr_get_ranked('%s:%s' % (index, query), hitset, self._get_ranking_params(), rows)
+ def get_ranked_result_sequence(self, query, index='fulltext', rows=ROWS):
+ ranked_result = solr_get_ranked('%s:%s' % (index, query),
+ HITSETS[query],
+ {'cutoff_amount': 10000,
+ 'cutoff_time_ms': 2000
+ },
+ rows)
return tuple([pair[0] for pair in ranked_result[0]])
- def _get_ranked_topN(self, n):
- return get_topN(n, self._RANKED)
-
- _RANKED = {
- 'Willnotfind': tuple(),
- 'higgs': (79, 51, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85),
- 'of': (50, 61, 60, 54, 56, 53, 10, 68, 44, 57, 83, 95, 92, 91, 74, 45, 48, 62, 82,
- 49, 51, 89, 90, 96, 43, 8, 64, 97, 15, 85, 78, 46, 55, 79, 84, 88, 81, 52,
- 58, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94),
- '"higgs boson"': (55, 56),
- }
-
- def _get_ranking_params(self, cutoff_amount=10000, cutoff_time=2000):
- """
- Default values from template_word_similarity_solr.cfg
- """
- return {
- 'cutoff_amount': cutoff_amount,
- 'cutoff_time_ms': cutoff_time
- }
+ def setUp(self):
+ init_Solr()
@nottest
def test_get_ranked(self):
"""solrutils - ranking results"""
- all_ranked = 0
- ranked_top = self._get_ranked_topN(all_ranked)
- self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind'))
- self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs'))
- self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of'))
- self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"'))
-
- @nottest
- def test_get_ranked_top(self):
- """solrutils - ranking top results"""
- top_n = 0
- self.assertEqual(tuple(), self._get_ranked_result_sequence(query='Willnotfind', rows=top_n))
- self.assertEqual(tuple(), self._get_ranked_result_sequence(query='higgs', rows=top_n))
- self.assertEqual(tuple(), self._get_ranked_result_sequence(query='of', rows=top_n))
- self.assertEqual(tuple(), self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n))
-
- top_n = 2
- ranked_top = self._get_ranked_topN(top_n)
- self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind', rows=top_n))
- self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs', rows=top_n))
- self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of', rows=top_n))
- self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n))
-
- top_n = 10
- ranked_top = self._get_ranked_topN(top_n)
- self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind', rows=top_n))
- self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs', rows=top_n))
- self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of', rows=top_n))
- self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n))
-
- @nottest
- def test_get_ranked_smaller_hitset(self):
- """solrutils - ranking smaller hitset"""
- hitset = intbitset.intbitset([47, 56, 58, 68, 85, 89])
- self.assertEqual((47, 56, 58, 68, 89, 85), self._get_ranked_result_sequence(query='higgs', hitset=hitset))
-
- hitset = intbitset.intbitset([45, 50, 61, 74, 94])
- self.assertEqual((50, 61, 74, 45, 94), self._get_ranked_result_sequence(query='of', hitset=hitset))
- self.assertEqual((74, 45, 94), self._get_ranked_result_sequence(query='of', hitset=hitset, rows=3))
-
- @nottest
- def test_get_ranked_larger_hitset(self):
- """solrutils - ranking larger hitset"""
- hitset = intbitset.intbitset([47, 56, 58, 68, 85, 89])
- self.assertEqual(tuple(), self._get_ranked_result_sequence(query='Willnotfind', hitset=hitset))
-
- hitset = intbitset.intbitset([47, 56, 55, 56, 58, 68, 85, 89])
- self.assertEqual((55, 56), self._get_ranked_result_sequence(query='"higgs boson"', hitset=hitset))
+ self.assertEqual(self.get_ranked_result_sequence(query='Willnotfind'), tuple())
+ self.assertEqual(self.get_ranked_result_sequence(query='of'), (8, 2, 1, 10, 7))
+ self.assertEqual(self.get_ranked_result_sequence(query='"of the"'), (8, 10, 1, 2, 7))
class TestSolrSimilarToRecid(InvenioTestCase):
@@ -172,69 +122,37 @@ class TestSolrSimilarToRecid(InvenioTestCase):
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- ./bibrank -w wrd for all records
"""
-
- def _get_similar_result_sequence(self, recid, rows=ROWS):
- similar_result = solr_get_similar_ranked(recid, self._all_records, self._get_similar_ranking_params(), rows)
+ def get_similar_result_sequence(self, recid, rows=ROWS):
+ similar_result = solr_get_similar_ranked(recid,
+ self._all_records,
+ {'cutoff_amount': 10000,
+ 'cutoff_time_ms': 2000,
+ 'find_similar_to_recid': {
+ 'more_results_factor': 5,
+ 'mlt_fl': 'mlt',
+ 'mlt_mintf': 0,
+ 'mlt_mindf': 0,
+ 'mlt_minwl': 0,
+ 'mlt_maxwl': 0,
+ 'mlt_maxqt': 25,
+ 'mlt_maxntp': 1000,
+ 'mlt_boost': 'false'
+ }
+ },
+ rows)
return tuple([pair[0] for pair in similar_result[0]])[-rows:]
- def _get_similar_topN(self, n):
- return get_topN(n, self._SIMILAR)
-
- _SIMILAR = {
- 30: (12, 95, 85, 82, 44, 1, 89, 64, 58, 15, 96, 61, 50, 86, 78, 77, 65, 62, 60,
- 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54,
- 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14,
- 59, 6, 10, 32, 33, 29, 30),
- 59: (17, 69, 3, 20, 109, 14, 22, 33, 28, 24, 60, 6, 73, 113, 5, 107, 78, 4, 13,
- 8, 45, 72, 74, 46, 104, 63, 71, 44, 87, 70, 103, 92, 57, 49, 7, 88, 68, 77,
- 62, 10, 93, 2, 65, 55, 43, 94, 96, 1, 11, 99, 91, 61, 51, 15, 64, 97, 89, 101,
- 108, 80, 86, 90, 54, 95, 102, 47, 100, 79, 83, 48, 12, 81, 82, 58, 50, 56, 84,
- 85, 53, 52, 59)
- }
-
- def _get_similar_ranking_params(self, cutoff_amount=10000, cutoff_time=2000):
- """
- Default values from template_word_similarity_solr.cfg
- """
- return {
- 'cutoff_amount': cutoff_amount,
- 'cutoff_time_ms': cutoff_time,
- 'find_similar_to_recid': {
- 'more_results_factor': 5,
- 'mlt_fl': 'mlt',
- 'mlt_mintf': 0,
- 'mlt_mindf': 0,
- 'mlt_minwl': 0,
- 'mlt_maxwl': 0,
- 'mlt_maxqt': 25,
- 'mlt_maxntp': 1000,
- 'mlt_boost': 'false'
- }
- }
-
_all_records = get_collection_reclist(CFG_SITE_NAME)
+ def setUp(self):
+ init_Solr()
+
@nottest
def test_get_similar_ranked(self):
"""solrutils - similar results"""
- all_ranked = 0
- similar_top = self._get_similar_topN(all_ranked)
- recid = 30
- self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid))
- recid = 59
- self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid))
-
- @nottest
- def test_get_similar_ranked_top(self):
- """solrutils - similar top results"""
- top_n = 5
- similar_top = self._get_similar_topN(top_n)
- recid = 30
- self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid, rows=top_n))
- recid = 59
- self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid, rows=top_n))
+ self.assertEqual(self.get_similar_result_sequence(1), (5, 4, 7, 8, 3, 6, 2, 10, 1))
+ self.assertEqual(self.get_similar_result_sequence(8), (3, 6, 9, 7, 2, 4, 5, 1, 10, 8))
class TestSolrWebSearch(InvenioTestCase):
@@ -242,31 +160,24 @@ class TestSolrWebSearch(InvenioTestCase):
make install-solrutils
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
- AND EITHER
- Solr index built: ./bibindex -w fulltext for all records
- OR
- WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- and ./bibrank -w wrd for all records
"""
+ def setUp(self):
+ init_Solr()
@nottest
def test_get_result(self):
"""solrutils - web search results"""
self.assertEqual([],
test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3AWillnotfind&rg=100',
- expected_text="[]"))
-
- self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rg=100',
- expected_text="[12, 47, 48, 51, 52, 55, 56, 58, 68, 79, 80, 81, 85, 89, 96]"))
+ expected_text='[]'))
self.assertEqual([],
test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rg=100',
- expected_text="[8, 10, 11, 12, 15, 43, 44, 45, 46, 47, 48, 49, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 64, 68, 74, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97]"))
+ expected_text='[1, 2, 7, 8, 10]'))
self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22higgs+boson%22&rg=100',
- expected_text="[12, 47, 51, 55, 56, 68, 81, 85]"))
+ test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22of+the%22&rg=100',
+ expected_text='[1, 2, 7, 8, 10]'))
class TestSolrWebRanking(InvenioTestCase):
@@ -274,42 +185,25 @@ class TestSolrWebRanking(InvenioTestCase):
make install-solrutils
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
- AND EITHER
- Solr index built: ./bibindex -w fulltext for all records
- OR
- WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- and ./bibrank -w wrd for all records
+ WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
"""
+ def setUp(self):
+ init_Solr()
@nottest
def test_get_ranked(self):
"""solrutils - web ranking results"""
self.assertEqual([],
test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3AWillnotfind&rg=100&rm=wrd',
- expected_text="[]"))
+ expected_text='[]'))
- self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rm=wrd',
- expected_text="[12, 51, 79, 80, 81, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85]"))
-
- self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rg=100&rm=wrd',
- expected_text="[12, 80, 81, 79, 51, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85]"))
-
- # Record 77 is restricted
self.assertEqual([],
test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rm=wrd',
- expected_text="[8, 10, 15, 43, 44, 45, 46, 48, 49, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 64, 68, 74, 78, 79, 81, 82, 83, 84, 85, 88, 89, 90, 91, 92, 95, 96, 97, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94]",
- username='admin'))
+ expected_text='[8, 2, 1, 10, 7]'))
self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rg=100&rm=wrd',
- expected_text="[61, 60, 54, 56, 53, 10, 68, 44, 57, 83, 95, 92, 91, 74, 45, 48, 62, 82, 49, 51, 89, 90, 96, 43, 8, 64, 97, 15, 85, 78, 46, 55, 79, 84, 88, 81, 52, 58, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94]",
- username='admin'))
-
- self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22higgs+boson%22&rg=100&rm=wrd',
- expected_text="[12, 47, 51, 68, 81, 85, 55, 56]"))
+ test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22of+the%22&rg=100&rm=wrd',
+ expected_text='[8, 10, 1, 2, 7]'))
class TestSolrWebSimilarToRecid(InvenioTestCase):
@@ -318,19 +212,20 @@ class TestSolrWebSimilarToRecid(InvenioTestCase):
CFG_SOLR_URL set
fulltext index in idxINDEX containing 'SOLR' in indexer column
WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
- ./bibrank -w wrd for all records
"""
+ def setUp(self):
+ init_Solr()
@nottest
def test_get_similar_ranked(self):
"""solrutils - web similar results"""
self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A30&rm=wrd',
- expected_text="[1, 3, 4, 8, 9, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 34, 43, 44, 49, 50, 56, 58, 61, 64, 66, 67, 69, 71, 73, 75, 76, 77, 78, 82, 85, 86, 87, 89, 90, 95, 96, 98, 104, 107, 109, 113, 65, 62, 60, 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54, 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14, 59, 6, 10, 32, 33, 29, 30]"))
+ test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A1&rm=wrd',
+ expected_text='[9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 113, 5, 4, 7, 8, 3, 6, 2, 10, 1]'))
self.assertEqual([],
- test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A30&rg=100&rm=wrd',
- expected_text="[3, 4, 8, 9, 13, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 34, 43, 49, 56, 66, 67, 69, 71, 73, 75, 76, 87, 90, 98, 104, 107, 109, 113, 12, 95, 85, 82, 44, 1, 89, 64, 58, 15, 96, 61, 50, 86, 78, 77, 65, 62, 60, 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54, 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14, 59, 6, 10, 32, 33, 29, 30]"))
+ test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A8&rg=100&rm=wrd',
+ expected_text='[11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 113, 3, 6, 9, 7, 2, 4, 5, 1, 10, 8]'))
class TestSolrLoadLogicalFieldSettings(InvenioTestCase):
@@ -339,7 +234,6 @@ class TestSolrLoadLogicalFieldSettings(InvenioTestCase):
CFG_SOLR_URL set
WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
"""
-
@nottest
def test_load_logical_fields(self):
"""solrutils - load logical fields"""
@@ -359,7 +253,6 @@ class TestSolrBuildFieldContent(InvenioTestCase):
CFG_SOLR_URL set
WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg
"""
-
@nottest
def test_build_default_field_content(self):
"""solrutils - build default field content"""
@@ -382,7 +275,6 @@ def test_build_custom_field_content(self):
self.assertEqual(u"""In 1962, CERN hosted the 11th International Conference on High Energy Physics. Among the distinguished visitors were eight Nobel prizewinners.Left to right: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang and Robert Hofstadter. En 1962, le CERN est l'hote de la onzieme Conference Internationale de Physique des Hautes Energies. Parmi les visiteurs eminents se trouvaient huit laureats du prix Nobel.De gauche a droite: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang et Robert Hofstadter.""",
get_field_content_in_utf8(6, 'abstract', tags))
-
TESTS = []
diff --git a/modules/miscutil/lib/testutils.py b/modules/miscutil/lib/testutils.py
index f1773104c2..2c3ea644d6 100644
--- a/modules/miscutil/lib/testutils.py
+++ b/modules/miscutil/lib/testutils.py
@@ -223,7 +223,7 @@ def get_authenticated_mechanize_browser(username="guest", password=""):
browser.submit()
username_account_page_body = browser.response().read()
try:
- username_account_page_body.index("You are logged in as %s." % username)
+ username_account_page_body.index("You are logged in as %s." % ("" + cgi.escape(username) + " "),)
except ValueError:
raise InvenioTestUtilsBrowserException('ERROR: Cannot login as %s.' % username)
return browser
diff --git a/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py b/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py
new file mode 100644
index 0000000000..bc01d22971
--- /dev/null
+++ b/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2012 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.dbquery import run_sql
+
+depends_on = ['invenio_release_1_1_0']
+
+def info():
+ return "New database tables for the WebNews module"
+
+def do_upgrade():
+ query_story = \
+"""DROP TABLE IF EXISTS `nwsSTORY`;
+CREATE TABLE `nwsSTORY` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `title` varchar(256) NOT NULL,
+ `body` text NOT NULL,
+ `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (`id`)
+);"""
+ run_sql(query_story)
+
+ query_tag = \
+"""DROP TABLE IF EXISTS `nwsTAG`;
+CREATE TABLE `nwsTAG` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `tag` varchar(64) NOT NULL,
+ PRIMARY KEY (`id`)
+);"""
+ run_sql(query_tag)
+
+ query_tooltip = \
+"""DROP TABLE IF EXISTS `nwsTOOLTIP`;
+CREATE TABLE `nwsTOOLTIP` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `id_story` int(11) NOT NULL,
+ `body` varchar(512) NOT NULL,
+ `target_element` varchar(256) NOT NULL DEFAULT '',
+ `target_page` varchar(256) NOT NULL DEFAULT '',
+ PRIMARY KEY (`id`),
+ KEY `id_story` (`id_story`),
+ CONSTRAINT `nwsTOOLTIP_ibfk_1` FOREIGN KEY (`id_story`) REFERENCES `nwsSTORY` (`id`)
+);"""
+ run_sql(query_tooltip)
+
+ query_story_tag = \
+"""DROP TABLE IF EXISTS `nwsSTORY_nwsTAG`;
+CREATE TABLE `nwsSTORY_nwsTAG` (
+ `id_story` int(11) NOT NULL,
+ `id_tag` int(11) NOT NULL,
+ PRIMARY KEY (`id_story`,`id_tag`),
+ KEY `id_story` (`id_story`),
+ KEY `id_tag` (`id_tag`),
+ CONSTRAINT `nwsSTORY_nwsTAG_ibfk_1` FOREIGN KEY (`id_story`) REFERENCES `nwsSTORY` (`id`),
+ CONSTRAINT `nwsSTORY_nwsTAG_ibfk_2` FOREIGN KEY (`id_tag`) REFERENCES `nwsTAG` (`id`)
+);"""
+ run_sql(query_story_tag)
+
+def estimate():
+ return 1
+
+def pre_upgrade():
+ pass
+
+def post_upgrade():
+ pass
diff --git a/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py b/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py
new file mode 100644
index 0000000000..163daf8b79
--- /dev/null
+++ b/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.dbquery import run_sql
+
+depends_on = ['invenio_release_1_1_0']
+
+def info():
+ return "New is_active column for the user_query_basket table"
+
+def do_upgrade():
+ run_sql("""
+ALTER TABLE user_query_basket ADD COLUMN is_active BOOL DEFAULT 1 AFTER notification;
+""")
+
+def estimate():
+ """ Estimate running time of upgrade in seconds (optional). """
+ return 1
diff --git a/modules/miscutil/lib/upgrades/invenio_2014_07_30_webcomment_new_column_body_format.py b/modules/miscutil/lib/upgrades/invenio_2014_07_30_webcomment_new_column_body_format.py
new file mode 100644
index 0000000000..7e4fc17257
--- /dev/null
+++ b/modules/miscutil/lib/upgrades/invenio_2014_07_30_webcomment_new_column_body_format.py
@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.dbquery import run_sql
+from invenio.webcomment_config import CFG_WEBCOMMENT_BODY_FORMATS
+from invenio.webmessage_mailutils import email_quoted_txt2html
+
+depends_on = ['invenio_release_1_1_0']
+
+
+def info():
+ return "New column 'body_format' for WebComment's cmtRECORDCOMMENT."
+
+
+def do_upgrade():
+ # First, insert the new column in the table.
+ cmtRECORDCOMMENT_definition = run_sql("SHOW CREATE TABLE cmtRECORDCOMMENT")[0][1]
+ if "body_format" not in cmtRECORDCOMMENT_definition:
+ run_sql("""ALTER TABLE cmtRECORDCOMMENT
+ ADD COLUMN body_format VARCHAR(10) NOT NULL DEFAULT %s
+ AFTER body;""",
+ (CFG_WEBCOMMENT_BODY_FORMATS["TEXT"],)
+ )
+
+ number_of_comments = run_sql("""SELECT COUNT(id)
+ FROM cmtRECORDCOMMENT""")[0][0]
+
+ if number_of_comments > 0:
+
+ # NOTE: Consider that the bigger the number of comments,
+ # the more powerful the server. Keep the number of
+ # batches fixed and scale the batch size instead.
+ number_of_select_batches = 100
+
+ select_batch_size = \
+ number_of_comments >= (number_of_select_batches * number_of_select_batches) and \
+ number_of_comments / number_of_select_batches or \
+ number_of_comments
+
+ number_of_select_iterations = \
+ number_of_select_batches + \
+ (number_of_comments % select_batch_size and 1)
+
+ comments_select_query = """ SELECT id,
+ body,
+ body_format
+ FROM cmtRECORDCOMMENT
+ LIMIT %s, %s"""
+
+ comments_update_query = """ UPDATE cmtRECORDCOMMENT
+ SET body = %s,
+ body_format = %s
+ WHERE id = %s"""
+
+ for number_of_select_iteration in xrange(number_of_select_iterations):
+
+ comments = run_sql(
+ comments_select_query,
+ (number_of_select_iteration * select_batch_size,
+ select_batch_size)
+ )
+
+ for (comment_id, comment_body, comment_body_format) in comments:
+
+ if comment_body_format == CFG_WEBCOMMENT_BODY_FORMATS["TEXT"]:
+
+ comment_body = email_quoted_txt2html(
+ comment_body,
+ indent_html=("", " ")
+ )
+
+ run_sql(
+ comments_update_query,
+ (comment_body,
+ CFG_WEBCOMMENT_BODY_FORMATS["HTML"],
+ comment_id)
+ )
+
+
+def estimate():
+ # TODO: The estimated time needed depends on the size of the table.
+ # Should we calculate this more accurately?
+ return 1
+
+
+def pre_upgrade():
+ pass
+
+
+def post_upgrade():
+ pass
diff --git a/modules/miscutil/lib/upgrades/invenio_2014_08_19_comment_to_bibdoc_relation_table.py b/modules/miscutil/lib/upgrades/invenio_2014_08_19_comment_to_bibdoc_relation_table.py
new file mode 100644
index 0000000000..f0ac2874d9
--- /dev/null
+++ b/modules/miscutil/lib/upgrades/invenio_2014_08_19_comment_to_bibdoc_relation_table.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2014 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.dbquery import run_sql
+
+depends_on = ['invenio_release_1_1_0']
+
+
+def info():
+ """Upgrader info."""
+ return ("Create a new table to be used for relating comment ids and "
+ "their record ids with bibdoc ids")
+
+
+def do_upgrade():
+ """Perform upgrade."""
+ run_sql("""
+CREATE TABLE IF NOT EXISTS `cmtRECORDCOMMENT_bibdoc` (
+ `id_bibrec` mediumint(8) unsigned NOT NULL,
+ `id_cmtRECORDCOMMENT` int(15) unsigned NOT NULL,
+ `id_bibdoc` mediumint(9) unsigned NOT NULL,
+ `version` tinyint(4) unsigned NOT NULL,
+ PRIMARY KEY (`id_bibrec`,`id_cmtRECORDCOMMENT`),
+ KEY `id_cmtRECORDCOMMENT` (`id_cmtRECORDCOMMENT`),
+ KEY `id_bibdoc` (`id_bibdoc`)
+) ENGINE=MyISAM;""")
+
+
+def estimate():
+ """ Estimate running time of upgrade in seconds (optional). """
+ return 1
diff --git a/modules/miscutil/lib/upgrades/invenio_2014_10_09_author_autocompletion.py b/modules/miscutil/lib/upgrades/invenio_2014_10_09_author_autocompletion.py
new file mode 100644
index 0000000000..57dbd8bc13
--- /dev/null
+++ b/modules/miscutil/lib/upgrades/invenio_2014_10_09_author_autocompletion.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2014 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Upgrade recipe for new column tag.recjson_value."""
+
+import os
+
+from invenio.dbquery import run_sql
+from invenio.config import CFG_PREFIX
+
+depends_on = ['invenio_release_1_1_0']
+
+
+def info():
+ """Upgrade recipe information."""
+ return "Set up autocompletion for DEMOTHE authors"
+
+
+def do_upgrade():
+ """Upgrade recipe procedure."""
+ os.system("cd %(prefix)s/var/www/js && \
+ wget https://cdnjs.cloudflare.com/ajax/libs/handlebars.js/1.3.0/handlebars.min.js && \
+ wget https://twitter.github.com/typeahead.js/releases/0.10.5/typeahead.bundle.min.js && \
+ wget https://raw.githubusercontent.com/es-shims/es5-shim/v4.0.3/es5-shim.min.js && \
+ wget https://raw.githubusercontent.com/es-shims/es5-shim/v4.0.3/es5-shim.map"
+ % {'prefix': CFG_PREFIX})
+
+ # Remove "one line per author" info on author textbox
+ run_sql("""UPDATE sbmFIELD
+ set fitext='* Author of the Thesis:
Invenio Access Control Engine can be called from within your Python programs
via both a regular Python API and CLI.
-In addition the you get an explanation of the program flow.
+In addition to the above features, you also get an explanation of the program flow.
Contents:
1. Regular API
diff --git a/modules/webaccess/lib/access_control_config.py b/modules/webaccess/lib/access_control_config.py
index 2479071b6f..86388c621b 100644
--- a/modules/webaccess/lib/access_control_config.py
+++ b/modules/webaccess/lib/access_control_config.py
@@ -1,5 +1,5 @@
# This file is part of Invenio.
-# Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 CERN.
+# Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -76,7 +76,7 @@ class InvenioWebAccessFireroleError(Exception):
CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS = ['8560_f']
if CFG_CERN_SITE:
- CFG_ACC_GRANT_VIEWER_RIGHTS_TO_EMAILS_IN_TAGS = ['506__m']
+ CFG_ACC_GRANT_VIEWER_RIGHTS_TO_EMAILS_IN_TAGS = ['506__m', '5061_d']
else:
CFG_ACC_GRANT_VIEWER_RIGHTS_TO_EMAILS_IN_TAGS = []
@@ -654,7 +654,7 @@ class InvenioWebAccessFireroleError(Exception):
'runbibeditmulti' : (_("Run Multi-Record Editor"), "%s/%s/multiedit/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
'runbibdocfile' : (_("Run Document File Manager"), "%s/%s/managedocfiles?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
'runbibmerge' : (_("Run Record Merger"), "%s/%s/merge/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
- 'runbibswordclient' : (_("Run BibSword client"), "%s/bibsword/?ln=%%s" % CFG_SITE_URL),
+ 'runbibswordclient' : (_("Run BibSword client"), "%s/sword_client/submissions?ln=%%s" % CFG_SITE_URL),
'cfgbibknowledge' : (_("Configure BibKnowledge"), "%s/kb?ln=%%s" % CFG_SITE_URL),
'cfgbibformat' : (_("Configure BibFormat"), "%s/admin/bibformat/bibformatadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgoaiharvest' : (_("Configure OAI Harvest"), "%s/admin/oaiharvest/oaiharvestadmin.py?ln=%%s" % CFG_SITE_URL),
diff --git a/modules/webalert/doc/admin/webalert-admin-guide.webdoc b/modules/webalert/doc/admin/webalert-admin-guide.webdoc
index 570843348b..5ba2a2d305 100644
--- a/modules/webalert/doc/admin/webalert-admin-guide.webdoc
+++ b/modules/webalert/doc/admin/webalert-admin-guide.webdoc
@@ -53,7 +53,7 @@ module to permit this functionality.
Configuring Alert Queries
Users may set up alert queries for example from their search history pages.
+href="/yoursearches/display">search history pages.
Administrators may edit existing users' alerts by modifying the
user_query_basket table. (There is no web interface yet
diff --git a/modules/webalert/lib/alert_engine.py b/modules/webalert/lib/alert_engine.py
index fb35bf5944..1e8f91b34b 100644
--- a/modules/webalert/lib/alert_engine.py
+++ b/modules/webalert/lib/alert_engine.py
@@ -63,15 +63,23 @@ def update_date_lastrun(alert):
return run_sql('update user_query_basket set date_lastrun=%s where id_user=%s and id_query=%s and id_basket=%s;', (strftime("%Y-%m-%d"), alert[0], alert[1], alert[2],))
-def get_alert_queries(frequency):
- """Return all the queries for the given frequency."""
-
- return run_sql('select distinct id, urlargs from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.frequency=%s and uqb.date_lastrun <= now();', (frequency,))
-
-def get_alert_queries_for_user(uid):
- """Returns all the queries for the given user id."""
-
- return run_sql('select distinct id, urlargs, uqb.frequency from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.id_user=%s and uqb.date_lastrun <= now();', (uid,))
+def get_alert_queries(frequency, only_active=True):
+ """
+ Return all the active alert queries for the given frequency.
+ If only_active is False: fetch all the alert queries
+ regardless of them being active or not.
+ """
+
+ return run_sql('select distinct id, urlargs from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.frequency=%%s and uqb.date_lastrun <= now()%s;' % (only_active and ' and is_active = 1' or '', ), (frequency, ))
+
+def get_alert_queries_for_user(uid, only_active=True):
+ """
+ Returns all the active alert queries for the given user id.
+ If only_active is False: fetch all the alert queries
+ regardless of them being active or not.
+ """
+
+ return run_sql('select distinct id, urlargs, uqb.frequency from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.id_user=%%s and uqb.date_lastrun <= now()%s;' % (only_active and ' and is_active = 1' or '', ), (uid, ))
def get_alerts(query, frequency):
"""Returns a dictionary of all the records found for a specific query and frequency along with other informationm"""
diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py
index 8fd46db718..5b44943144 100644
--- a/modules/webalert/lib/webalert.py
+++ b/modules/webalert/lib/webalert.py
@@ -21,20 +21,22 @@
import cgi
import time
+from urllib import quote
from invenio.config import CFG_SITE_LANG
from invenio.dbquery import run_sql
-from invenio.webuser import isGuestUser
from invenio.errorlib import register_exception
-from invenio.webaccount import warning_guest_user
-from invenio.webbasket import create_personal_baskets_selection_box
-from invenio.webbasket_dblayer import check_user_owns_baskets
+from invenio.webbasket_dblayer import \
+ check_user_owns_baskets, \
+ get_all_user_personal_basket_ids_by_topic
from invenio.messages import gettext_set_language
-from invenio.dateutils import convert_datestruct_to_datetext, convert_datetext_to_dategui
+from invenio.dateutils import convert_datestruct_to_datetext
import invenio.template
webalert_templates = invenio.template.load('webalert')
+CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS = 20
+
### IMPLEMENTATION
class AlertError(Exception):
@@ -54,8 +56,10 @@ def check_alert_name(alert_name, uid, ln=CFG_SITE_LANG):
raise AlertError( _("You already have an alert named %s.") % ('' + cgi.escape(alert_name) + ' ',) )
def get_textual_query_info_from_urlargs(urlargs, ln=CFG_SITE_LANG):
- """Return nicely formatted search pattern and catalogue from urlargs of the search query.
- Suitable for 'your searches' display."""
+ """
+ Return nicely formatted search pattern and catalogue from urlargs of the search query.
+ """
+
out = ""
args = cgi.parse_qs(urlargs)
return webalert_templates.tmpl_textual_query_info_from_urlargs(
@@ -64,71 +68,6 @@ def get_textual_query_info_from_urlargs(urlargs, ln=CFG_SITE_LANG):
)
return out
-def perform_display(permanent, uid, ln=CFG_SITE_LANG):
- """display the searches performed by the current user
- input: default permanent="n"; permanent="y" display permanent queries(most popular)
- output: list of searches in formatted html
- """
- # load the right language
- _ = gettext_set_language(ln)
-
- # first detect number of queries:
- nb_queries_total = 0
- nb_queries_distinct = 0
- query = "SELECT COUNT(*),COUNT(DISTINCT(id_query)) FROM user_query WHERE id_user=%s"
- res = run_sql(query, (uid,), 1)
- try:
- nb_queries_total = res[0][0]
- nb_queries_distinct = res[0][1]
- except:
- pass
-
- # query for queries:
- params = ()
- if permanent == "n":
- SQL_query = "SELECT DISTINCT(q.id),q.urlargs "\
- "FROM query q, user_query uq "\
- "WHERE uq.id_user=%s "\
- "AND uq.id_query=q.id "\
- "ORDER BY q.id DESC"
- params = (uid,)
- else:
- # permanent="y"
- SQL_query = "SELECT q.id,q.urlargs "\
- "FROM query q "\
- "WHERE q.type='p'"
- query_result = run_sql(SQL_query, params)
-
- queries = []
- if len(query_result) > 0:
- for row in query_result :
- if permanent == "n":
- res = run_sql("SELECT DATE_FORMAT(MAX(date),'%%Y-%%m-%%d %%H:%%i:%%s') FROM user_query WHERE id_user=%s and id_query=%s",
- (uid, row[0]))
- try:
- lastrun = res[0][0]
- except:
- lastrun = _("unknown")
- else:
- lastrun = ""
- queries.append({
- 'id' : row[0],
- 'args' : row[1],
- 'textargs' : get_textual_query_info_from_urlargs(row[1], ln=ln),
- 'lastrun' : lastrun,
- })
-
-
- return webalert_templates.tmpl_display_alerts(
- ln = ln,
- permanent = permanent,
- nb_queries_total = nb_queries_total,
- nb_queries_distinct = nb_queries_distinct,
- queries = queries,
- guest = isGuestUser(uid),
- guesttxt = warning_guest_user(type="alerts", ln=ln)
- )
-
def check_user_can_add_alert(id_user, id_query):
"""Check if ID_USER has really alert adding rights on ID_QUERY
(that is, the user made the query herself or the query is one of
@@ -148,7 +87,16 @@ def check_user_can_add_alert(id_user, id_query):
return True
return False
-def perform_input_alert(action, id_query, alert_name, frequency, notification, id_basket, uid, old_id_basket=None, ln = CFG_SITE_LANG):
+def perform_input_alert(action,
+ id_query,
+ alert_name,
+ frequency,
+ notification,
+ id_basket,
+ uid,
+ is_active,
+ old_id_basket = None,
+ ln = CFG_SITE_LANG):
"""get the alert settings
input: action="add" for a new alert (blank form), action="modify" for an update
(get old values)
@@ -156,21 +104,30 @@ def perform_input_alert(action, id_query, alert_name, frequency, notification, i
for the "modify" action specify old alert_name, frequency of checking,
e-mail notification and basket id.
output: alert settings input form"""
+
# load the right language
_ = gettext_set_language(ln)
+
# security check:
if not check_user_can_add_alert(uid, id_query):
raise AlertError(_("You do not have rights for this operation."))
+
+ # normalize is_active (it should be either 1 (True) or 0 (False))
+ is_active = is_active and 1 or 0
+
# display query information
res = run_sql("SELECT urlargs FROM query WHERE id=%s", (id_query,))
try:
urlargs = res[0][0]
except:
urlargs = "UNKNOWN"
- baskets = create_personal_baskets_selection_box(uid=uid,
- html_select_box_name='idb',
- selected_bskid=old_id_basket,
- ln=ln)
+
+ baskets = webalert_templates.tmpl_personal_basket_select_element(
+ bskid = old_id_basket,
+ personal_baskets_list = get_all_user_personal_basket_ids_by_topic(uid),
+ select_element_name = "idb",
+ ln = ln)
+
return webalert_templates.tmpl_input_alert(
ln = ln,
query = get_textual_query_info_from_urlargs(urlargs, ln = ln),
@@ -182,9 +139,7 @@ def perform_input_alert(action, id_query, alert_name, frequency, notification, i
old_id_basket = old_id_basket,
id_basket = id_basket,
id_query = id_query,
- guest = isGuestUser(uid),
- guesttxt = warning_guest_user(type="alerts", ln=ln)
- )
+ is_active = is_active)
def check_alert_is_unique(id_basket, id_query, uid, ln=CFG_SITE_LANG ):
"""check the user does not have another alert for the specified query and basket"""
@@ -236,57 +191,186 @@ def perform_add_alert(alert_name, frequency, notification,
run_sql(query, params)
out = _("The alert %s has been added to your profile.")
out %= '' + cgi.escape(alert_name) + ' '
- out += perform_list_alerts(uid, ln=ln)
+ out += perform_request_youralerts_display(uid, idq=0, ln=ln)
return out
-def perform_list_alerts(uid, ln=CFG_SITE_LANG):
- """perform_list_alerts display the list of alerts for the connected user"""
+def perform_request_youralerts_display(uid,
+ idq=0,
+ page=1,
+ step=CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS,
+ p='',
+ ln=CFG_SITE_LANG):
+ """
+ Display a list of the user defined alerts. If a specific query id is defined
+ only the user alerts based on that query appear.
+
+ @param uid: The user id
+ @type uid: int
+
+ @param idq: The specified query id for which to display the user alerts
+ @type idq: int
+
+ @param page:
+ @type page: integer
+
+ @param step:
+ @type step: integer
+
+ @param ln: The interface language
+ @type ln: string
+
+ @return: HTML formatted list of the user defined alerts.
+ """
+
# set variables
out = ""
- # query the database
- query = """ SELECT q.id, q.urlargs,
- a.id_basket, b.name,
- a.alert_name, a.frequency,a.notification,
- DATE_FORMAT(a.date_creation,'%%Y-%%m-%%d %%H:%%i:%%s'),
- DATE_FORMAT(a.date_lastrun,'%%Y-%%m-%%d %%H:%%i:%%s')
- FROM user_query_basket a LEFT JOIN query q ON a.id_query=q.id
- LEFT JOIN bskBASKET b ON a.id_basket=b.id
- WHERE a.id_user=%s
- ORDER BY a.alert_name ASC """
- res = run_sql(query, (uid,))
- alerts = []
- for (qry_id, qry_args,
- bsk_id, bsk_name,
- alrt_name, alrt_frequency, alrt_notification, alrt_creation, alrt_last_run) in res:
- try:
- if not qry_id:
- raise StandardError("""\
+ if idq:
+ idq_clause = "q.id=%i" % (idq,)
+ else:
+ idq_clause = ""
+
+ search_clause = ""
+ search_clause_urlargs = []
+ search_clause_alert_name = []
+ if p:
+ p_stripped_args = p.split()
+ sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg).replace('%','%%') + '%%\'' for p_stripped_arg in p_stripped_args]
+ for sql_p_stripped_arg in sql_p_stripped_args:
+ search_clause_urlargs.append("q.urlargs LIKE %s" % (sql_p_stripped_arg,))
+ search_clause_alert_name.append("uqb.alert_name LIKE %s" % (sql_p_stripped_arg,))
+ search_clause = "((%s) OR (%s))" % (" AND ".join(search_clause_urlargs),
+ " AND ".join(search_clause_alert_name))
+
+ query_nb_alerts = """ SELECT COUNT(IF((uqb.id_user=%%s
+ %s),uqb.id_query,NULL)),
+ COUNT(q.id)
+ FROM user_query_basket AS uqb
+ RIGHT JOIN query AS q
+ ON uqb.id_query=q.id
+ %s""" % ((search_clause and ' AND ' + search_clause),
+ (idq_clause and ' WHERE ' + idq_clause))
+ params_nb_alerts = (uid,)
+ result_nb_alerts = run_sql(query_nb_alerts, params_nb_alerts)
+ nb_alerts = result_nb_alerts[0][0]
+ nb_queries = result_nb_alerts[0][1]
+
+ # In case we do have some alerts, proceed with the needed calculations and
+ # fetching them from the database
+ if nb_alerts:
+ # The real page starts counting from 0, i.e. minus 1 from the human page
+ real_page = page - 1
+ # The step needs to be a positive integer
+ if (step <= 0):
+ step = CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS
+ # The maximum real page is the integer division of the total number of
+ # searches and the searches displayed per page
+ max_real_page = nb_alerts and ((nb_alerts / step) - (not (nb_alerts % step) and 1 or 0))
+ # Check if the selected real page exceeds the maximum real page and reset
+ # if needed
+ if (real_page >= max_real_page):
+ #if ((nb_queries_distinct % step) != 0):
+ # real_page = max_real_page
+ #else:
+ # real_page = max_real_page - 1
+ real_page = max_real_page
+ page = real_page + 1
+ elif (real_page < 0):
+ real_page = 0
+ page = 1
+ # Calculate the start value for the SQL LIMIT constraint
+ limit_start = real_page * step
+ # Calculate the display of the paging navigation arrows for the template
+ paging_navigation = (real_page >= 2,
+ real_page >= 1,
+ real_page <= (max_real_page - 1),
+ (real_page <= (max_real_page - 2)) and (max_real_page + 1))
+
+ query = """ SELECT q.id,
+ q.urlargs,
+ uqb.id_basket,
+ bsk.name,
+ uqb.alert_name,
+ uqb.frequency,
+ uqb.notification,
+ DATE_FORMAT(uqb.date_creation,'%s'),
+ DATE_FORMAT(uqb.date_lastrun,'%s'),
+ uqb.is_active
+ FROM user_query_basket uqb
+ LEFT JOIN query q
+ ON uqb.id_query=q.id
+ LEFT JOIN bskBASKET bsk
+ ON uqb.id_basket=bsk.id
+ WHERE uqb.id_user=%%s
+ %s
+ %s
+ ORDER BY uqb.alert_name ASC
+ LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s',
+ '%%Y-%%m-%%d %%H:%%i:%%s',
+ (idq_clause and ' AND ' + idq_clause),
+ (search_clause and ' AND ' + search_clause))
+ params = (uid, limit_start, step)
+ result = run_sql(query, params)
+
+ alerts = []
+ for (query_id,
+ query_args,
+ bsk_id,
+ bsk_name,
+ alert_name,
+ alert_frequency,
+ alert_notification,
+ alert_creation,
+ alert_last_run,
+ alert_is_active) in result:
+ try:
+ if not query_id:
+ raise StandardError("""\
Warning: I have detected a bad alert for user id %d.
It seems one of his/her alert queries was deleted from the 'query' table.
Please check this and delete it if needed.
Otherwise no problem, I'm continuing with the other alerts now.
-Here are all the alerts defined by this user: %s""" % (uid, repr(res)))
- alerts.append({
- 'queryid' : qry_id,
- 'queryargs' : qry_args,
- 'textargs' : get_textual_query_info_from_urlargs(qry_args, ln=ln),
- 'userid' : uid,
- 'basketid' : bsk_id,
- 'basketname' : bsk_name,
- 'alertname' : alrt_name,
- 'frequency' : alrt_frequency,
- 'notification' : alrt_notification,
- 'created' : convert_datetext_to_dategui(alrt_creation),
- 'lastrun' : convert_datetext_to_dategui(alrt_last_run)
- })
- except StandardError:
- register_exception(alert_admin=True)
-
- # link to the "add new alert" form
- out = webalert_templates.tmpl_list_alerts(ln=ln, alerts=alerts,
- guest=isGuestUser(uid),
- guesttxt=warning_guest_user(type="alerts", ln=ln))
+Here are all the alerts defined by this user: %s""" % (uid, repr(result)))
+ alerts.append({'queryid' : query_id,
+ 'queryargs' : query_args,
+ 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln),
+ 'userid' : uid,
+ 'basketid' : bsk_id,
+ 'basketname' : bsk_name,
+ 'alertname' : alert_name,
+ 'frequency' : alert_frequency,
+ 'notification' : alert_notification,
+ 'created' : alert_creation,
+ 'lastrun' : alert_last_run,
+ 'is_active' : alert_is_active})
+ except StandardError:
+ register_exception(alert_admin=True)
+ else:
+ alerts = []
+ paging_navigation = ()
+
+ # check if there are any popular alerts already defined
+ query_popular_alerts_p = """ SELECT COUNT(q.id)
+ FROM query q
+ WHERE q.type='p'"""
+ result_popular_alerts_p = run_sql(query_popular_alerts_p)
+ if result_popular_alerts_p[0][0] > 0:
+ popular_alerts_p = True
+ else:
+ popular_alerts_p = False
+
+ out = webalert_templates.tmpl_youralerts_display(
+ ln = ln,
+ alerts = alerts,
+ nb_alerts = nb_alerts,
+ nb_queries = nb_queries,
+ idq = idq,
+ page = page,
+ step = step,
+ paging_navigation = paging_navigation,
+ p = p,
+ popular_alerts_p = popular_alerts_p)
+
return out
def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG):
@@ -314,11 +398,94 @@ def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG)
out += "The alert %s has been removed from your profile.%s .
%s
' % _('Alert successfully paused.')
+ else:
+ out += '%s
' % _('Unable to pause alert.')
+
+ out += perform_request_youralerts_display(uid, idq=0, ln=ln)
+
return out
+def perform_resume_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG):
+ """Resume an alert
+ input: alert name
+ identifier of the query;
+ identifier of the basket
+ uid
+ output: confirmation message + the list of alerts Web page"""
+
+ # load the right language
+ _ = gettext_set_language(ln)
+
+ # security check:
+ if not check_user_can_add_alert(uid, id_query):
+ raise AlertError(_("You do not have rights for this operation."))
+
+ # set variables
+ out = ""
+ if (None in (alert_name, id_query, id_basket, uid)):
+ return out
+
+ # DB call to resume the alert
+ query = """ UPDATE user_query_basket
+ SET is_active = 1
+ WHERE id_user=%s
+ AND id_query=%s
+ AND id_basket=%s"""
+ params = (uid, id_query, id_basket)
+ res = run_sql(query, params)
+
+ if res:
+ out += '%s
' % _('Alert successfully resumed.')
+ else:
+ out += '%s
' % _('Unable to resume alert.')
+
+ out += perform_request_youralerts_display(uid, idq=0, ln=ln)
-def perform_update_alert(alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid, ln = CFG_SITE_LANG):
+ return out
+
+def perform_update_alert(alert_name,
+ frequency,
+ notification,
+ id_basket,
+ id_query,
+ old_id_basket,
+ uid,
+ is_active,
+ ln = CFG_SITE_LANG):
"""update alert settings into the database
input: the name of the new alert;
alert frequency: 'month', 'week' or 'day';
@@ -330,9 +497,12 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer
output: confirmation message + the list of alerts Web page"""
out = ''
# sanity check
- if (None in (alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid)):
+ if (None in (alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid, is_active)):
return out
+ # normalize is_active (it should be either 1 (True) or 0 (False))
+ is_active = is_active and 1 or 0
+
# load the right language
_ = gettext_set_language(ln)
@@ -363,18 +533,25 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer
check_alert_is_unique( id_basket, id_query, uid, ln)
# update a row into the alerts table: user_query_basket
- query = """UPDATE user_query_basket
- SET alert_name=%s,frequency=%s,notification=%s,
- date_creation=%s,date_lastrun='',id_basket=%s
- WHERE id_user=%s AND id_query=%s AND id_basket=%s"""
+ query = """ UPDATE user_query_basket
+ SET alert_name=%s,
+ frequency=%s,
+ notification=%s,
+ date_creation=%s,
+ date_lastrun='',
+ id_basket=%s,
+ is_active=%s
+ WHERE id_user=%s
+ AND id_query=%s
+ AND id_basket=%s"""
params = (alert_name, frequency, notification,
convert_datestruct_to_datetext(time.localtime()),
- id_basket, uid, id_query, old_id_basket)
+ id_basket, is_active, uid, id_query, old_id_basket)
run_sql(query, params)
out += _("The alert %s has been successfully updated.") % ("" + cgi.escape(alert_name) + " ",)
- out += "' % ln, 'x_url_close': ' '}
- return out
+ # fetch the popular queries
+ query = """ SELECT q.id,
+ q.urlargs
+ FROM query q
+ WHERE q.type='p'"""
+ result = run_sql(query)
+
+ search_queries = []
+ if result:
+ for search_query in result:
+ search_query_id = search_query[0]
+ search_query_args = search_query[1]
+ search_queries.append({'id' : search_query_id,
+ 'args' : search_query_args,
+ 'textargs' : get_textual_query_info_from_urlargs(search_query_args, ln=ln)})
+
+ return webalert_templates.tmpl_youralerts_popular(ln = ln,
+ search_queries = search_queries)
+
+def count_user_alerts_for_given_query(id_user,
+ id_query):
+ """
+ Count the alerts the user has defined based on a specific query.
+
+ @param user_id: The user id.
+ @type user_id: integer
+
+ @param user_id: The query id.
+ @type user_id: integer
+
+ @return: The number of alerts.
+ """
+
+ query = """ SELECT COUNT(id_query)
+ FROM user_query_basket
+ WHERE id_user=%s
+ AND id_query=%s"""
+ params = (id_user, id_query)
+ result = run_sql(query, params)
+
+ return result[0][0]
diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py
index 80e75bf05b..389c1472db 100644
--- a/modules/webalert/lib/webalert_templates.py
+++ b/modules/webalert/lib/webalert_templates.py
@@ -19,19 +19,25 @@
import cgi
import time
+from urlparse import parse_qs
+from datetime import date as datetime_date
from invenio.config import \
CFG_WEBALERT_ALERT_ENGINE_EMAIL, \
CFG_SITE_NAME, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_SITE_URL, \
+ CFG_SITE_LANG, \
+ CFG_SITE_SECURE_URL, \
CFG_WEBALERT_MAX_NUM_OF_RECORDS_IN_ALERT_EMAIL, \
CFG_SITE_RECORD
from invenio.messages import gettext_set_language
from invenio.htmlparser import get_as_text, wrap, wrap_records
from invenio.urlutils import create_html_link
-
from invenio.search_engine import guess_primary_collection_of_a_record, get_coll_ancestors
+from invenio.dateutils import convert_datetext_to_datestruct
+from invenio.webbasket_dblayer import get_basket_ids_and_names
+from invenio.webbasket_config import CFG_WEBBASKET_CATEGORIES
class Template:
def tmpl_errorMsg(self, ln, error_msg, rest = ""):
@@ -93,40 +99,43 @@ def tmpl_textual_query_info_from_urlargs(self, ln, args):
out += "" + _("Collection") + ": " + "; ".join(args['cc']) + "' % (CFG_SITE_SECURE_URL, ln),
+ 'x_alerts_number': str(alerts),
+ 'x_url_close' : ' ' % (CFG_SITE_SECURE_URL, ln),
+ 'x_search_interface_url_open' : ' ' % (CFG_SITE_URL, ln),
+ 'x_url_close' : ' ',}
+
return out
- def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notification,
- baskets, old_id_basket, id_basket, id_query,
- guest, guesttxt):
+ def tmpl_input_alert(self,
+ ln,
+ query,
+ alert_name,
+ action,
+ frequency,
+ notification,
+ baskets,
+ old_id_basket,
+ id_query,
+ is_active):
"""
Displays an alert adding form.
@@ -148,13 +157,9 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio
- 'old_id_basket' *string* - The id of the previous basket of this alert
- - 'id_basket' *string* - The id of the basket of this alert
-
- 'id_query' *string* - The id of the query associated to this alert
- - 'guest' *bool* - If the user is a guest user
-
- - 'guesttxt' *string* - The HTML content of the warning box for guest users (produced by webaccount.tmpl_warning_guest_user)
+ - 'is_active' *boolean* - is the alert active or not
"""
# load the right message language
@@ -193,6 +198,10 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio
+ %(is_active_label)s
+
%(send_email)s
@@ -205,7 +214,8 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio
%(store_basket)s
- %(baskets)s
+ %(baskets)s
+
""" % {
'action': action,
'alert_name' : _("Alert identification name:"),
@@ -225,270 +235,345 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio
'specify' : _("if %(x_fmt_open)sno%(x_fmt_close)s you must specify a basket") % {'x_fmt_open': '',
'x_fmt_close': ' '},
'store_basket' : _("Store results in basket?"),
- 'baskets': baskets
+ 'baskets': baskets,
+ 'is_active_label' : _("Is the alert active?"),
+ 'is_active_checkbox' : is_active and 'checked="checked" ' or '',
}
- out += """
-
-
+ out += """
-
-
-
- """ % {
- 'idq' : id_query,
- 'ln' : ln,
- 'set_alert' : _("SET ALERT"),
- 'clear_data' : _("CLEAR DATA"),
- }
+
""" % {'idq' : id_query,
+ 'ln' : ln,
+ 'set_alert' : _("SET ALERT"),
+ 'clear_data' : _("CLEAR DATA"),}
+
if action == "update":
out += '' + _("Set a new alert from %(x_url1_open)syour searches%(x_url1_close)s, the %(x_url2_open)spopular searches%(x_url2_close)s, or the input form.") + '
'
- out %= {'x_url1_open': '',
- 'x_url1_close': ' ',
- 'x_url2_open': '',
- 'x_url2_close': ' ',
- }
- if len(alerts):
- out += """
-
- %(no)s
- %(name)s
- %(search_freq)s
- %(notification)s
- %(result_basket)s
- %(date_run)s
- %(date_created)s
- %(query)s
- %(action)s """ % {
- 'no' : _("No"),
- 'name' : _("Name"),
- 'search_freq' : _("Search checking frequency"),
- 'notification' : _("Notification by email"),
- 'result_basket' : _("Result in basket"),
- 'date_run' : _("Date last run"),
- 'date_created' : _("Creation date"),
- 'query' : _("Query"),
- 'action' : _("Action"),
- }
- i = 0
- for alert in alerts:
- i += 1
- if alert['frequency'] == "day":
- frequency = _("daily")
+ # In case the user has not yet defined any alerts display only the
+ # following message
+ if not nb_alerts:
+ if idq and not p:
+ if nb_queries:
+ msg = _('You have not defined any alerts yet based on that search query.')
+ msg += "%s ' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')),
+ 'youralerts': '%s ' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))}
else:
- if alert['frequency'] == "week":
- frequency = _("weekly")
- else:
- frequency = _("monthly")
-
- if alert['notification'] == "y":
- notification = _("yes")
+ msg = _('The selected search query seems to be invalid.')
+ msg += "%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ elif p and not idq:
+ msg = _('You have not defined any alerts yet including the terms %s.') % \
+ ('' + cgi.escape(p) + ' ',)
+ msg += "%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ elif p and idq:
+ if nb_queries:
+ msg = _('You have not defined any alerts yet based on that search query including the terms %s.') % \
+ ('' + cgi.escape(p) + ' ',)
+ msg += "%s ' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')),
+ 'youralerts': '%s ' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))}
else:
- notification = _("no")
-
- # we clean up the HH:MM part of lastrun, since it is always 00:00
- lastrun = alert['lastrun'].split(',')[0]
- created = alert['created'].split(',')[0]
-
- out += """
- #%(index)d
- %(alertname)s
- %(frequency)s
- %(notification)s
- %(basketname)s
- %(lastrun)s
- %(created)s
- %(textargs)s
-
- %(remove_link)s%(search)s
-
- """ % {
- 'index' : i,
- 'alertname' : cgi.escape(alert['alertname']),
- 'frequency' : frequency,
- 'notification' : notification,
- 'basketname' : alert['basketname'] and cgi.escape(alert['basketname']) \
- or "- " + _("no basket") + " -",
- 'lastrun' : lastrun,
- 'created' : created,
- 'textargs' : alert['textargs'],
- 'queryid' : alert['queryid'],
- 'basketid' : alert['basketid'],
- 'freq' : alert['frequency'],
- 'notif' : alert['notification'],
- 'ln' : ln,
- 'modify_link': create_html_link("./modify",
- {'ln': ln,
- 'idq': alert['queryid'],
- 'name': alert['alertname'],
- 'freq': frequency,
- 'notif':notification,
- 'idb':alert['basketid'],
- 'old_idb':alert['basketid']},
- _("Modify")),
- 'remove_link': create_html_link("./remove",
- {'ln': ln,
- 'idq': alert['queryid'],
- 'name': alert['alertname'],
- 'idb':alert['basketid']},
- _("Remove")),
- 'siteurl' : CFG_SITE_URL,
- 'search' : _("Execute search"),
- 'queryargs' : cgi.escape(alert['queryargs'])
- }
-
- out += '
'
-
- out += '' + (_("You have defined %s alerts.") % ('' + str(len(alerts)) + ' ' )) + '
'
- if guest:
- out += guesttxt
- return out
-
- def tmpl_display_alerts(self, ln, permanent, nb_queries_total, nb_queries_distinct, queries, guest, guesttxt):
- """
- Displays the list of alerts
-
- Parameters:
-
- - 'ln' *string* - The language to display the interface in
-
- - 'permanent' *string* - If displaying most popular searches ('y') or only personal searches ('n')
-
- - 'nb_queries_total' *string* - The number of personal queries in the last period
-
- - 'nb_queries_distinct' *string* - The number of distinct queries in the last period
-
- - 'queries' *array* - The existing queries:
-
- - 'id' *string* - The id of the associated query
-
- - 'args' *string* - The query string
-
- - 'textargs' *string* - The textual description of the query string
-
- - 'lastrun' *string* - The last running date (only for personal queries)
-
- - 'guest' *bool* - If the user is a guest user
-
- - 'guesttxt' *string* - The HTML content of the warning box for guest users (produced by webaccount.tmpl_warning_guest_user)
- """
-
- # load the right message language
- _ = gettext_set_language(ln)
-
- if len(queries) == 0:
- out = _("You have not executed any search yet. Please go to the %(x_url_open)ssearch interface%(x_url_close)s first.") % \
- {'x_url_open': '',
- 'x_url_close': ' '}
+ msg = _('The selected search query seems to be invalid.')
+ msg += "%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ else:
+ msg = _('You have not defined any alerts yet.')
+ msg += '%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ out = '' + msg + '
'
return out
- out = ''
-
- # display message: number of items in the list
- if permanent == "n":
- msg = _("You have performed %(x_nb1)s searches (%(x_nb2)s different questions) during the last 30 days or so.") % {'x_nb1': nb_queries_total,
- 'x_nb2': nb_queries_distinct}
- out += '' + msg + '
'
+ # Diplay a message about the number of alerts.
+ if idq and not p:
+ msg = _('You have defined %(number_of_alerts)s alerts based on that search query.') % \
+ {'number_of_alerts': '' + str(nb_alerts) + ' '}
+ msg += '%s ' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')),
+ 'youralerts': '%s ' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))}
+ elif p and not idq:
+ msg = _('You have defined %(number_of_alerts)s alerts including the terms %(p)s.') % \
+ {'p': '' + cgi.escape(p) + ' ',
+ 'number_of_alerts': '' + str(nb_alerts) + ' '}
+ msg += '%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ elif idq and p:
+ msg = _('You have defined %(number_of_alerts)s alerts based on that search query including the terms %(p)s.') % \
+ {'p': '' + cgi.escape(p) + ' ',
+ 'number_of_alerts': '' + str(nb_alerts) + ' '}
+ msg += '%s ' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')),
+ 'youralerts': '%s ' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))}
else:
- # permanent="y"
- msg = _("Here are the %s most popular searches.")
- msg %= ('' + str(len(queries)) + ' ')
- out += '' + msg + '
'
+ msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \
+ {'number_of_alerts': '' + str(nb_alerts) + ' '}
+ msg += '%s ' % (CFG_SITE_SECURE_URL, ln, _('your searches')),
+ 'popular_alerts': popular_alerts_p and ', %s ' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '',
+ 'search_interface': '%s ' %(CFG_SITE_URL, ln, _('searching for something new'))}
+ out = '' + msg + '
'
+
+ # Search form
+ search_form = """
+
+ """ % {'search_text': _('Search all your alerts for'),
+ 'action': '%s/youralerts/display?ln=%s' % (CFG_SITE_SECURE_URL, ln),
+ 'p': cgi.escape(p),
+ 'submit_label': _('Search')}
+ out += '' + search_form + '
'
+
+ counter = (page - 1) * step
+ youralerts_display_html = ""
+ for alert in alerts:
+ counter += 1
+ alert_name = alert['alertname']
+ alert_query_id = alert['queryid']
+ alert_query_args = alert['queryargs']
+ # We don't need the text args, we'll use a local function to do a
+ # better job.
+ #alert_text_args = alert['textargs']
+ # We don't need the user id. The alerts page is a logged in user
+ # only page anyway.
+ #alert_user_id = alert['userid']
+ alert_basket_id = alert['basketid']
+ alert_basket_name = alert['basketname']
+ alert_frequency = alert['frequency']
+ alert_notification = alert['notification']
+ alert_creation_date = alert['created']
+ alert_last_run_date = alert['lastrun']
+ alert_active_p = alert['is_active']
+
+ alert_details_frequency = _('Runs') + ' ' + \
+ (alert_frequency == 'day' and '' + _('daily') + ' ' or \
+ alert_frequency == 'week' and '' + _('weekly') + ' ' or \
+ alert_frequency == 'month' and '' + _('monthly') + ' ')
+ alert_details_notification = alert_notification == 'y' and _('You are notified by e-mail ') or \
+ alert_notification == 'n' and ''
+ alert_details_basket = alert_basket_name and '%s %s · ' + \
+ alert_details_notification) + \
+ (alert_details_basket and \
+ ' · ' + \
+ alert_details_basket)
+
+ alert_details_search_query = get_html_user_friendly_alert_query_args(alert_query_args, ln)
+
+ alert_details_creation_date = get_html_user_friendly_date_from_datetext(alert_creation_date, True, False, ln)
+ alert_details_last_run_date = get_html_user_friendly_date_from_datetext(alert_last_run_date, True, False, ln)
+ alert_details_creation_last_run_dates = _('Last run:') + ' ' + \
+ alert_details_last_run_date + \
+ ' · ' + \
+ _('Created:') + ' ' + \
+ alert_details_creation_date
+
+ alert_details_options_pause_or_resume = create_html_link('%s/youralerts/%s' % \
+ (CFG_SITE_SECURE_URL, alert_active_p and 'pause' or 'resume'),
+ {'ln' : ln,
+ 'idq' : alert_query_id,
+ 'name' : alert_name,
+ 'idb' : alert_basket_id,},
+ alert_active_p and _('Pause') or _('Resume'))
+
+ alert_details_options_edit = create_html_link('%s/youralerts/modify' % \
+ (CFG_SITE_SECURE_URL,),
+ {'ln' : ln,
+ 'idq' : alert_query_id,
+ 'name' : alert_name,
+ 'freq' : alert_frequency,
+ 'notif' : alert_notification,
+ 'idb' : alert_basket_id,
+ 'is_active' : alert_active_p,
+ 'old_idb' : alert_basket_id},
+ _('Edit'))
+
+ alert_details_options_delete = create_html_link('%s/youralerts/remove' % \
+ (CFG_SITE_SECURE_URL,),
+ {'ln' : ln,
+ 'idq' : alert_query_id,
+ 'name' : alert_name,
+ 'idb' : alert_basket_id},
+ _('Delete'),
+ {'onclick': 'return confirm(\'%s\')' % \
+ (_('Are you sure you want to permanently delete this alert?'),)})
+
+ # TODO: find a nice way to format the display alert options
+ alert_details_options = '· ' + \
+ '· ' + \
+ '
+
+ %(counter)i.
+
+
+
+
%(warning_label_is_active_p)s %(alert_name_label)s %(alert_name)s
+
%(alert_details_search_query)s
+
%(alert_details_frequency_notification_basket)s
+
%(alert_details_creation_last_run_dates)s
+ %(alert_details_options)s
+
+ """ % {'counter': counter,
+ 'alert_name_label' : _('Alert'),
+ 'alert_name': cgi.escape(alert_name),
+ 'alert_details_frequency_notification_basket': alert_details_frequency_notification_basket,
+ 'alert_details_search_query': alert_details_search_query,
+ 'alert_details_options': alert_details_options,
+ 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates,
+ 'css_class_content_is_active_p' : not alert_active_p and ' youralerts_display_table_content_inactive' or '',
+ 'warning_label_is_active_p' : not alert_active_p and '[ %s ] ' % _('paused') or '',
+ }
- # display the list of searches
- out += """
-
- %(no)s
- %(question)s
- %(action)s """ % {
- 'no' : "#",
- 'question' : _("Question"),
- 'action' : _("Action")
- }
- if permanent == "n":
- out += '%s ' % _("Last Run")
- out += " \n"
- i = 0
- for query in queries :
- i += 1
- # id, pattern, base, search url and search set alert, date
- out += """
- #%(index)d
- %(textargs)s
- %(execute_query)s %(set_alert)s %s ' % query['lastrun']
- out += """ \n"""
- out += "
%i to %i from %i total alerts') % \
+ (displayed_alerts_from, displayed_alerts_to, nb_alerts)
+ paging_navigation_html += " "
+ if paging_navigation[2]:
+ paging_navigation_html += """
+
+
+
+ %(youralerts_display_html)s
+
+
""" % {'paging_navigation_html': paging_navigation_html,
+ 'youralerts_display_html': youralerts_display_html}
return out
@@ -641,7 +726,7 @@ def tmpl_alert_email_body(self, name, description, url, records, pattern,
%s Alert Service <%s>
Unsubscribe? See <%s>
Need human intervention? Contact <%s>
-''' % (CFG_SITE_NAME, CFG_SITE_URL, CFG_SITE_URL + '/youralerts/list', CFG_SITE_SUPPORT_EMAIL)
+''' % (CFG_SITE_NAME, CFG_SITE_URL, CFG_SITE_URL + '/youralerts/display', CFG_SITE_SUPPORT_EMAIL)
return body
@@ -656,3 +741,271 @@ def tmpl_alert_email_record(self, recid=0, xml_record=None):
out = wrap_records(get_as_text(xml_record=xml_record))
# TODO: add Detailed record url for external records?
return out
+
+ def tmpl_youralerts_popular(self,
+ ln,
+ search_queries):
+ """
+ Display the popular alerts.
+
+ Parameters:
+
+ - 'ln' *string* - The language to display the interface in
+
+ - 'search_queries' *array* - The existing queries:
+
+ - 'id' *string* - The id of the associated query
+
+ - 'args' *string* - The query string
+
+ - 'textargs' *string* - The textual description of the query string
+ """
+
+ # load the right message language
+ _ = gettext_set_language(ln)
+
+ if not search_queries:
+ out = _("There are no popular alerts defined yet.")
+ return out
+
+ out = ''
+
+ # display the list of searches
+ out += """
+
+ %(no)s
+ %(question)s
+ %(action)s """ % {
+ 'no' : "#",
+ 'question' : _("Question"),
+ 'action' : _("Action")
+ }
+ out += " \n"
+ i = 0
+ for search_query in search_queries :
+ i += 1
+ # id, pattern, base, search url and search set alert
+ out += """
+ #%(index)d
+ %(textargs)s
+ %(execute_query)s %(set_alert)s \n"""
+ out += "
""" % (select_element_name,)
+
+ # Calculate the selected basket if there is one pre-selected.
+ bskid = bskid and str(bskid) or ""
+
+ # Create the default disabled label option.
+ out += """
+ %(label)s """ % \
+ {'value': 0,
+ 'selected': bskid == '' and ' selected="selected"' or '',
+ 'label': _("Don't store results in basket...")}
+
+ # Create the optgroups and options for the user personal baskets.
+ if personal_baskets_list:
+ out += """
+ """ % ('* ' + _('Your personal baskets') + ' *',)
+ for baskets_topic_and_bskids in personal_baskets_list:
+ topic = baskets_topic_and_bskids[0]
+ bskids = baskets_topic_and_bskids[1].split(',')
+ out += """
+ """ % (cgi.escape(topic, True),)
+ bskids_and_names = get_basket_ids_and_names(bskids)
+ for bskid_and_name in bskids_and_names:
+ basket_value = str(bskid_and_name[0])
+ basket_name = bskid_and_name[1]
+ out += """
+ %(label)s """ % \
+ {'value': basket_value,
+ 'selected': basket_value == bskid and ' selected="selected"' or '',
+ 'label': cgi.escape(basket_name, True)}
+ out += """
+ """
+ out += """
+ """
+
+ out += """
+ """
+
+ return out
+
+def get_html_user_friendly_alert_query_args(args,
+ ln=CFG_SITE_LANG):
+ """
+ Internal function.
+ Returns an HTML formatted user friendly description of a search query's
+ arguments.
+
+ @param args: The search query arguments as they apear in the search URL
+ @type args: string
+
+ @param ln: The language to display the interface in
+ @type ln: string
+
+ @return: HTML formatted user friendly description of a search query's
+ arguments
+ """
+
+ # Load the right language
+ _ = gettext_set_language(ln)
+
+ # Arguments dictionary
+ args_dict = parse_qs(args)
+
+ if not args_dict.has_key('p') and not args_dict.has_key('p1') and not args_dict.has_key('p2') and not args_dict.has_key('p3'):
+ search_patterns_html = _('Searching for everything')
+ else:
+ search_patterns_html = _('Searching for') + ' '
+ if args_dict.has_key('p'):
+ search_patterns_html += '' + cgi.escape(args_dict['p'][0]) + ' '
+ if args_dict.has_key('f'):
+ search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f'][0]) + ' '
+ if args_dict.has_key('p1'):
+ if args_dict.has_key('p'):
+ search_patterns_html += ' ' + _('and') + ' '
+ search_patterns_html += '' + cgi.escape(args_dict['p1'][0]) + ' '
+ if args_dict.has_key('f1'):
+ search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f1'][0]) + ' '
+ if args_dict.has_key('p2'):
+ if args_dict.has_key('p') or args_dict.has_key('p1'):
+ if args_dict.has_key('op1'):
+ search_patterns_html += ' %s ' % (args_dict['op1'][0] == 'a' and _('and') or \
+ args_dict['op1'][0] == 'o' and _('or') or \
+ args_dict['op1'][0] == 'n' and _('and not') or
+ ', ',)
+ search_patterns_html += '' + cgi.escape(args_dict['p2'][0]) + ' '
+ if args_dict.has_key('f2'):
+ search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f2'][0]) + ' '
+ if args_dict.has_key('p3'):
+ if args_dict.has_key('p') or args_dict.has_key('p1') or args_dict.has_key('p2'):
+ if args_dict.has_key('op2'):
+ search_patterns_html += ' %s ' % (args_dict['op2'][0] == 'a' and _('and') or \
+ args_dict['op2'][0] == 'o' and _('or') or \
+ args_dict['op2'][0] == 'n' and _('and not') or
+ ', ',)
+ search_patterns_html += '' + cgi.escape(args_dict['p3'][0]) + ' '
+ if args_dict.has_key('f3'):
+ search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f3'][0]) + ' '
+
+ if not args_dict.has_key('c') and not args_dict.has_key('cc'):
+ collections_html = _('in all the collections')
+ else:
+ collections_html = _('in the following collection(s)') + ': '
+ if args_dict.has_key('c'):
+ collections_html += ', '.join('' + cgi.escape(collection) + ' ' for collection in args_dict['c'])
+ elif args_dict.has_key('cc'):
+ collections_html += '' + cgi.escape(args_dict['cc'][0]) + ' '
+
+ search_query_args_html = search_patterns_html + '%(account)s """ % {
- 'sitesecureurl' : CFG_SITE_SECURE_URL,
- 'ln': argd['ln'],
- 'account' : _("Your Account"),
- },
- description=_("%s Personalize, Display searches") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
- keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
- uid=uid,
- language=argd['ln'],
- req=req,
- lastupdated=__lastupdated__,
- navmenuid='youralerts',
- secure_page_p=1)
+ redirect_to_url(req, '%s/youralerts/display' % (CFG_SITE_SECURE_URL,))
def input(self, req, form):
@@ -139,9 +105,16 @@ def input(self, req, form):
text = _("You are not authorized to use alerts."))
try:
- html = webalert.perform_input_alert("add", argd['idq'], argd['name'], argd['freq'],
- argd['notif'], argd['idb'], uid, ln=argd['ln'])
- except webalert.AlertError, msg:
+ html = perform_input_alert("add",
+ argd['idq'],
+ argd['name'],
+ argd['freq'],
+ argd['notif'],
+ argd['idb'],
+ uid,
+ is_active = 1,
+ ln = argd['ln'])
+ except AlertError, msg:
return page(title=_("Error"),
body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
navtrail= """%(account)s """ % {
@@ -198,6 +171,7 @@ def modify(self, req, form):
'freq': (str, "week"),
'notif': (str, "y"),
'idb': (int, 0),
+ 'is_active': (int, 0),
'error_msg': (str, ""),
})
@@ -224,9 +198,16 @@ def modify(self, req, form):
text = _("You are not authorized to use alerts."))
try:
- html = webalert.perform_input_alert("update", argd['idq'], argd['name'], argd['freq'],
- argd['notif'], argd['idb'], uid, argd['old_idb'], ln=argd['ln'])
- except webalert.AlertError, msg:
+ html = perform_input_alert("update", argd['idq'],
+ argd['name'],
+ argd['freq'],
+ argd['notif'],
+ argd['idb'],
+ uid,
+ argd['is_active'],
+ argd['old_idb'],
+ ln=argd['ln'])
+ except AlertError, msg:
return page(title=_("Error"),
body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
navtrail= """%(account)s """ % {
@@ -275,21 +256,25 @@ def modify(self, req, form):
lastupdated=__lastupdated__,
navmenuid='youralerts')
- def list(self, req, form):
+ def display(self, req, form):
- argd = wash_urlargd(form, {})
+ argd = wash_urlargd(form, {'idq': (int, 0),
+ 'page': (int, 1),
+ 'step': (int, 20),
+ 'p': (str, ''),
+ 'ln': (str, '')})
uid = getUid(req)
if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
- return page_not_authorized(req, "%s/youralerts/list" % \
+ return page_not_authorized(req, "%s/youralerts/display" % \
(CFG_SITE_SECURE_URL,),
navmenuid="youralerts")
elif uid == -1 or isGuestUser(uid):
return redirect_to_url(req, "%s/youraccount/login%s" % (
CFG_SITE_SECURE_URL,
make_canonical_urlargd({
- 'referer' : "%s/youralerts/list%s" % (
+ 'referer' : "%s/youralerts/display%s" % (
CFG_SITE_SECURE_URL,
make_canonical_urlargd(argd, {})),
"ln" : argd['ln']}, {})))
@@ -307,17 +292,21 @@ def list(self, req, form):
else:
user_str = ""
try:
- register_customevent("alerts", ["list", "", user_str])
+ register_customevent("alerts", ["display", "", user_str])
except:
register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'")
return page(title=_("Your Alerts"),
- body=webalert.perform_list_alerts(uid, ln = argd['ln']),
- navtrail= """%(account)s """ % {
- 'sitesecureurl' : CFG_SITE_SECURE_URL,
- 'ln': argd['ln'],
- 'account' : _("Your Account"),
- },
+ body=perform_request_youralerts_display(uid,
+ idq=argd['idq'],
+ page=argd['page'],
+ step=argd['step'],
+ p=argd['p'],
+ ln=argd['ln']),
+ navtrail= """%(account)s """ % \
+ {'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account")},
description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
uid=uid,
@@ -358,9 +347,9 @@ def add(self, req, form):
text = _("You are not authorized to use alerts."))
try:
- html = webalert.perform_add_alert(argd['name'], argd['freq'], argd['notif'],
+ html = perform_add_alert(argd['name'], argd['freq'], argd['notif'],
argd['idb'], argd['idq'], uid, ln=argd['ln'])
- except webalert.AlertError, msg:
+ except AlertError, msg:
return page(title=_("Error"),
body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
navtrail= """%(account)s """ % {
@@ -409,6 +398,7 @@ def update(self, req, form):
'notif': (str, None),
'idb': (int, None),
'idq': (int, None),
+ 'is_active': (int, 0),
'old_idb': (int, None),
})
@@ -435,9 +425,16 @@ def update(self, req, form):
text = _("You are not authorized to use alerts."))
try:
- html = webalert.perform_update_alert(argd['name'], argd['freq'], argd['notif'],
- argd['idb'], argd['idq'], argd['old_idb'], uid, ln=argd['ln'])
- except webalert.AlertError, msg:
+ html = perform_update_alert(argd['name'],
+ argd['freq'],
+ argd['notif'],
+ argd['idb'],
+ argd['idq'],
+ argd['old_idb'],
+ uid,
+ argd['is_active'],
+ ln=argd['ln'])
+ except AlertError, msg:
return page(title=_("Error"),
body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
navtrail= """%(account)s """ % {
@@ -480,6 +477,9 @@ def update(self, req, form):
navmenuid='youralerts')
def remove(self, req, form):
+ """
+ Remove an alert from the DB.
+ """
argd = wash_urlargd(form, {'name': (str, None),
'idq': (int, None),
@@ -509,9 +509,9 @@ def remove(self, req, form):
text = _("You are not authorized to use alerts."))
try:
- html = webalert.perform_remove_alert(argd['name'], argd['idq'],
+ html = perform_remove_alert(argd['name'], argd['idq'],
argd['idb'], uid, ln=argd['ln'])
- except webalert.AlertError, msg:
+ except AlertError, msg:
return page(title=_("Error"),
body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
navtrail= """%(account)s """ % {
@@ -554,3 +554,227 @@ def remove(self, req, form):
req=req,
lastupdated=__lastupdated__,
navmenuid='youralerts')
+
+ def pause(self, req, form):
+ """
+ Pause an alert.
+ """
+
+ argd = wash_urlargd(form, {'name' : (str, None),
+ 'idq' : (int, None),
+ 'idb' : (int, None),
+ 'ln' : (str, CFG_SITE_LANG),
+ })
+
+ uid = getUid(req)
+
+ if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
+ return page_not_authorized(req, "%s/youralerts/pause" % \
+ (CFG_SITE_SECURE_URL,),
+ navmenuid="youralerts")
+ elif uid == -1 or isGuestUser(uid):
+ return redirect_to_url(req, "%s/youraccount/login%s" % (
+ CFG_SITE_SECURE_URL,
+ make_canonical_urlargd({
+ 'referer' : "%s/youralerts/pause%s" % (
+ CFG_SITE_SECURE_URL,
+ make_canonical_urlargd(argd, {})),
+ "ln" : argd['ln']}, {})))
+
+ # load the right language
+ _ = gettext_set_language(argd['ln'])
+ user_info = collect_user_info(req)
+ if not user_info['precached_usealerts']:
+ return page_not_authorized(req, "../", \
+ text = _("You are not authorized to use alerts."))
+
+ try:
+ html = perform_pause_alert(argd['name'],
+ argd['idq'],
+ argd['idb'],
+ uid,
+ ln=argd['ln'])
+ except AlertError, msg:
+ return page(title=_("Error"),
+ body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
+ navtrail= """%(account)s """ % {
+ 'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account"),
+ },
+ description=_("%s Personalize, Set a new alert") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ uid=uid,
+ language=argd['ln'],
+ req=req,
+ lastupdated=__lastupdated__,
+ navmenuid='youralerts')
+
+ # register event in webstat
+ alert_str = "%s (%d)" % (argd['name'], argd['idq'])
+ if user_info['email']:
+ user_str = "%s (%d)" % (user_info['email'], user_info['uid'])
+ else:
+ user_str = ""
+ try:
+ register_customevent("alerts", ["pause", alert_str, user_str])
+ except:
+ register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'")
+
+ # display success
+ return page(title=_("Display alerts"),
+ body=html,
+ navtrail= """%(account)s """ % {
+ 'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account"),
+ },
+ description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ uid=uid,
+ language=argd['ln'],
+ req=req,
+ lastupdated=__lastupdated__,
+ navmenuid='youralerts')
+
+ def resume(self, req, form):
+ """
+ Resume an alert.
+ """
+
+ argd = wash_urlargd(form, {'name' : (str, None),
+ 'idq' : (int, None),
+ 'idb' : (int, None),
+ 'ln' : (str, CFG_SITE_LANG),
+ })
+
+ uid = getUid(req)
+
+ if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
+ return page_not_authorized(req, "%s/youralerts/resume" % \
+ (CFG_SITE_SECURE_URL,),
+ navmenuid="youralerts")
+ elif uid == -1 or isGuestUser(uid):
+ return redirect_to_url(req, "%s/youraccount/login%s" % (
+ CFG_SITE_SECURE_URL,
+ make_canonical_urlargd({
+ 'referer' : "%s/youralerts/resume%s" % (
+ CFG_SITE_SECURE_URL,
+ make_canonical_urlargd(argd, {})),
+ "ln" : argd['ln']}, {})))
+
+ # load the right language
+ _ = gettext_set_language(argd['ln'])
+ user_info = collect_user_info(req)
+ if not user_info['precached_usealerts']:
+ return page_not_authorized(req, "../", \
+ text = _("You are not authorized to use alerts."))
+
+ try:
+ html = perform_resume_alert(argd['name'],
+ argd['idq'],
+ argd['idb'],
+ uid,
+ ln=argd['ln'])
+ except AlertError, msg:
+ return page(title=_("Error"),
+ body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg),
+ navtrail= """%(account)s """ % {
+ 'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account"),
+ },
+ description=_("%s Personalize, Set a new alert") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ uid=uid,
+ language=argd['ln'],
+ req=req,
+ lastupdated=__lastupdated__,
+ navmenuid='youralerts')
+
+ # register event in webstat
+ alert_str = "%s (%d)" % (argd['name'], argd['idq'])
+ if user_info['email']:
+ user_str = "%s (%d)" % (user_info['email'], user_info['uid'])
+ else:
+ user_str = ""
+ try:
+ register_customevent("alerts", ["resume", alert_str, user_str])
+ except:
+ register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'")
+
+ # display success
+ return page(title=_("Display alerts"),
+ body=html,
+ navtrail= """%(account)s """ % {
+ 'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account"),
+ },
+ description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ uid=uid,
+ language=argd['ln'],
+ req=req,
+ lastupdated=__lastupdated__,
+ navmenuid='youralerts')
+
+ def popular(self, req, form):
+ """
+ Display a list of popular alerts.
+ """
+
+ argd = wash_urlargd(form, {'ln': (str, "en")})
+
+ uid = getUid(req)
+
+ # load the right language
+ _ = gettext_set_language(argd['ln'])
+
+ if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
+ return page_not_authorized(req, "%s/youralerts/popular" % \
+ (CFG_SITE_SECURE_URL,),
+ navmenuid="youralerts")
+ elif uid == -1 or isGuestUser(uid):
+ return redirect_to_url(req, "%s/youraccount/login%s" % \
+ (CFG_SITE_SECURE_URL,
+ make_canonical_urlargd(
+ {'referer' : "%s/youralerts/popular%s" % (
+ CFG_SITE_SECURE_URL,
+ make_canonical_urlargd(argd, {})),
+ 'ln' : argd['ln']},
+ {})
+ )
+ )
+
+ user_info = collect_user_info(req)
+ if not user_info['precached_usealerts']:
+ return page_not_authorized(req, "../", \
+ text = _("You are not authorized to use alerts."))
+
+ # register event in webstat
+ if user_info['email']:
+ user_str = "%s (%d)" % (user_info['email'], user_info['uid'])
+ else:
+ user_str = ""
+ try:
+ register_customevent("alerts", ["popular", "", user_str])
+ except:
+ register_exception(
+ suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'")
+
+ return page(title=_("Popular Alerts"),
+ body = perform_request_youralerts_popular(ln=argd['ln']),
+ navtrail = """%(account)s """ % {
+ 'sitesecureurl' : CFG_SITE_SECURE_URL,
+ 'ln': argd['ln'],
+ 'account' : _("Your Account"),
+ },
+ description=_("%s Personalize, Popular Alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME),
+ uid=uid,
+ language=argd['ln'],
+ req=req,
+ lastupdated=__lastupdated__,
+ navmenuid='youralerts',
+ secure_page_p=1)
diff --git a/modules/webbasket/lib/webbasket.py b/modules/webbasket/lib/webbasket.py
index 0a193eabb2..fa24d325d7 100644
--- a/modules/webbasket/lib/webbasket.py
+++ b/modules/webbasket/lib/webbasket.py
@@ -1053,7 +1053,7 @@ def perform_request_search(uid,
p="",
b="",
n=0,
- #format='xm',
+ #record_format='xm',
ln=CFG_SITE_LANG):
"""Search the baskets...
@param uid: user id
@@ -1160,12 +1160,12 @@ def perform_request_search(uid,
# The search format for external records. This means in which format will
# the external records be fetched from the database to be searched then.
- format = 'xm'
+ record_format = 'xm'
### Calculate the search results for the user's personal baskets ###
if b.startswith("P") or not b:
personal_search_results = {}
- personal_items = db.get_all_items_in_user_personal_baskets(uid, selected_topic, format)
+ personal_items = db.get_all_items_in_user_personal_baskets(uid, selected_topic, record_format)
personal_local_items = personal_items[0]
personal_external_items = personal_items[1]
personal_external_items_xml_records = {}
@@ -1239,7 +1239,7 @@ def perform_request_search(uid,
### Calculate the search results for the user's group baskets ###
if b.startswith("G") or not b:
group_search_results = {}
- group_items = db.get_all_items_in_user_group_baskets(uid, selected_group_id, format)
+ group_items = db.get_all_items_in_user_group_baskets(uid, selected_group_id, record_format)
group_local_items = group_items[0]
group_external_items = group_items[1]
group_external_items_xml_records = {}
@@ -1328,7 +1328,7 @@ def perform_request_search(uid,
### Calculate the search results for the user's public baskets ###
if b.startswith("E") or not b:
public_search_results = {}
- public_items = db.get_all_items_in_user_public_baskets(uid, format)
+ public_items = db.get_all_items_in_user_public_baskets(uid, record_format)
public_local_items = public_items[0]
public_external_items = public_items[1]
public_external_items_xml_records = {}
@@ -1411,7 +1411,7 @@ def perform_request_search(uid,
### Calculate the search results for all the public baskets ###
if b.startswith("A"):
all_public_search_results = {}
- all_public_items = db.get_all_items_in_all_public_baskets(format)
+ all_public_items = db.get_all_items_in_all_public_baskets(record_format)
all_public_local_items = all_public_items[0]
all_public_external_items = all_public_items[1]
all_public_external_items_xml_records = {}
@@ -1695,14 +1695,15 @@ def perform_request_delete_note(uid,
#warnings_notes.append('WRN_WEBBASKET_DELETE_INVALID_NOTE')
warnings_html += webbasket_templates.tmpl_warnings(exc.message, ln)
- (body, warnings, navtrail) = perform_request_display(uid=uid,
- selected_category=category,
- selected_topic=topic,
- selected_group_id=group_id,
- selected_bskid=bskid,
- selected_recid=recid,
- of='hb',
- ln=CFG_SITE_LANG)
+ (body, dummy, navtrail) = perform_request_display(
+ uid=uid,
+ selected_category=category,
+ selected_topic=topic,
+ selected_group_id=group_id,
+ selected_bskid=bskid,
+ selected_recid=recid,
+ of='hb',
+ ln=CFG_SITE_LANG)
body = warnings_html + body
#warnings.extend(warnings_notes)
@@ -2042,7 +2043,7 @@ def perform_request_delete(uid, bskid, confirmed=0,
if not(db.check_user_owns_baskets(uid, [bskid])):
try:
raise InvenioWebBasketWarning(_('Sorry, you do not have sufficient rights on this basket.'))
- except InvenioWebBasketWarning, exc:
+ except InvenioWebBasketWarning:
register_exception(stream='warning')
#warnings.append(exc.message)
#warnings.append(('WRN_WEBBASKET_NO_RIGHTS',))
@@ -2111,7 +2112,7 @@ def perform_request_edit(uid, bskid, topic="", new_name='',
if rights != CFG_WEBBASKET_SHARE_LEVELS['MANAGE']:
try:
raise InvenioWebBasketWarning(_('Sorry, you do not have sufficient rights on this basket.'))
- except InvenioWebBasketWarning, exc:
+ except InvenioWebBasketWarning:
register_exception(stream='warning')
#warnings.append(exc.message)
#warnings.append(('WRN_WEBBASKET_NO_RIGHTS',))
@@ -2394,23 +2395,6 @@ def create_guest_warning_box(ln=CFG_SITE_LANG):
"""return a warning message about logging into system"""
return webbasket_templates.tmpl_create_guest_warning_box(ln)
-def create_personal_baskets_selection_box(uid,
- html_select_box_name='baskets',
- selected_bskid=None,
- ln=CFG_SITE_LANG):
- """Return HTML box for basket selection. Only for personal baskets.
- @param uid: user id
- @param html_select_box_name: name used in html form
- @param selected_bskid: basket currently selected
- @param ln: language
- """
- baskets = db.get_all_personal_baskets_names(uid)
- return webbasket_templates.tmpl_personal_baskets_selection_box(
- baskets,
- html_select_box_name,
- selected_bskid,
- ln)
-
def create_basket_navtrail(uid,
category=CFG_WEBBASKET_CATEGORIES['PRIVATE'],
topic="", group=0,
@@ -2487,7 +2471,7 @@ def create_basket_navtrail(uid,
if bskid:
basket = db.get_public_basket_infos(bskid)
if basket:
- basket_html = """ > """ % \
+ basket_html = """ > %s """ % \
(CFG_SITE_URL,
'category=' + category + '&ln=' + ln + \
'#bsk' + str(bskid),
@@ -2590,30 +2574,15 @@ def create_webbasket_navtrail(uid,
return out
-def account_list_baskets(uid, ln=CFG_SITE_LANG):
- """Display baskets informations on account page"""
- _ = gettext_set_language(ln)
+def account_user_baskets(uid, ln=CFG_SITE_LANG):
+ """
+ Display baskets informations on account page
+ """
+
(personal, group, external) = db.count_baskets(uid)
- link = '%s '
- base_url = CFG_SITE_URL + '/yourbaskets/display?category=%s&ln=' + ln
- personal_text = personal
- if personal:
- url = base_url % CFG_WEBBASKET_CATEGORIES['PRIVATE']
- personal_text = link % (url, personal_text)
- group_text = group
- if group:
- url = base_url % CFG_WEBBASKET_CATEGORIES['GROUP']
- group_text = link % (url, group_text)
- external_text = external
- if external:
- url = base_url % CFG_WEBBASKET_CATEGORIES['EXTERNAL']
- else:
- url = CFG_SITE_URL + '/yourbaskets/list_public_baskets?ln=' + ln
- external_text = link % (url, external_text)
- out = _("You have %(x_nb_perso)s personal baskets and are subscribed to %(x_nb_group)s group baskets and %(x_nb_public)s other users public baskets.") %\
- {'x_nb_perso': personal_text,
- 'x_nb_group': group_text,
- 'x_nb_public': external_text}
+
+ out = webbasket_templates.tmpl_account_user_baskets(personal, group, external, ln)
+
return out
def page_start(req, of='xm'):
diff --git a/modules/webbasket/lib/webbasket_dblayer.py b/modules/webbasket/lib/webbasket_dblayer.py
index 4dd7f37949..546b038158 100644
--- a/modules/webbasket/lib/webbasket_dblayer.py
+++ b/modules/webbasket/lib/webbasket_dblayer.py
@@ -37,113 +37,42 @@
from invenio.websession_config import CFG_WEBSESSION_USERGROUP_STATUS
from invenio.search_engine import get_fieldvalues
-########################### Table of contents ################################
-#
-# NB. functions preceeded by a star use usergroup table
-#
-# 1. General functions
-# - count_baskets
-# - check_user_owns_basket
-# - get_max_user_rights_on_basket
-#
-# 2. Personal baskets
-# - get_personal_baskets_info_for_topic
-# - get_all_personal_basket_ids_and_names_by_topic
-# - get_all_personal_baskets_names
-# - get_basket_name
-# - is_personal_basket_valid
-# - is_topic_valid
-# - get_basket_topic
-# - get_personal_topics_infos
-# - rename_basket
-# - rename_topic
-# - move_baskets_to_topic
-# - delete_basket
-# - create_basket
-#
-# 3. Actions on baskets
-# - get_basket_record
-# - get_basket_content
-# - get_basket_item
-# - get_basket_item_title_and_URL
-# - share_basket_with_group
-# - update_rights
-# - move_item
-# - delete_item
-# - add_to_basket
-# - get_external_records_by_collection
-# - store_external_records
-# - store_external_urls
-# - store_external_source
-# - get_external_colid_and_url
-#
-# 4. Group baskets
-# - get_group_basket_infos
-# - get_group_name
-# - get_all_group_basket_ids_and_names_by_group
-# - (*) get_all_group_baskets_names
-# - is_shared_to
-#
-# 5. External baskets (baskets user has subscribed to)
-# - get_external_baskets_infos
-# - get_external_basket_info
-# - get_all_external_basket_ids_and_names
-# - count_external_baskets
-# - get_all_external_baskets_names
-#
-# 6. Public baskets (interface to subscribe to baskets)
-# - get_public_basket_infos
-# - get_public_basket_info
-# - get_basket_general_infos
-# - get_basket_owner_id
-# - count_public_baskets
-# - get_public_baskets_list
-# - is_basket_public
-# - subscribe
-# - unsubscribe
-# - is_user_subscribed_to_basket
-# - count_subscribers
-# - (*) get_groups_subscribing_to_basket
-# - get_rights_on_public_basket
-#
-# 7. Annotating
-# - get_notes
-# - get_note
-# - save_note
-# - delete_note
-# - note_belongs_to_item_in_basket_p
-#
-# 8. Usergroup functions
-# - (*) get_group_infos
-# - count_groups_user_member_of
-# - (*) get_groups_user_member_of
-#
-# 9. auxilliary functions
-# - __wash_sql_count
-# - __decompress_last
-# - create_pseudo_record
-# - prettify_url
-
########################## General functions ##################################
def count_baskets(uid):
- """Return (nb personal baskets, nb group baskets, nb external
- baskets) tuple for given user"""
- query1 = "SELECT COUNT(id) FROM bskBASKET WHERE id_owner=%s"
- res1 = run_sql(query1, (int(uid),))
- personal = __wash_sql_count(res1)
- query2 = """SELECT count(ugbsk.id_bskbasket)
- FROM usergroup_bskBASKET ugbsk LEFT JOIN user_usergroup uug
- ON ugbsk.id_usergroup=uug.id_usergroup
- WHERE uug.id_user=%s AND uug.user_status!=%s
- GROUP BY ugbsk.id_usergroup"""
- params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
- res2 = run_sql(query2, params)
- if len(res2):
- groups = reduce(lambda x, y: x + y, map(lambda x: x[0], res2))
+ """
+ Return (number of personal baskets,
+ number of group baskets,
+ number of external baskets)
+ tuple for the given user based on their user id.
+ """
+
+ # TODO: Maybe put this in a try..except ?
+ uid = int(uid)
+
+ query_personal = """SELECT COUNT(id)
+ FROM bskBASKET
+ WHERE id_owner=%s"""
+ params_personal = (uid,)
+ res_personal = run_sql(query_personal, params_personal)
+ personal = __wash_sql_count(res_personal)
+
+ query_group = """ SELECT COUNT(ugbsk.id_bskbasket)
+ FROM usergroup_bskBASKET ugbsk
+ LEFT JOIN user_usergroup uug
+ ON ugbsk.id_usergroup = uug.id_usergroup
+ WHERE uug.id_user = %s
+ AND uug.user_status != %s
+ GROUP BY ugbsk.id_usergroup"""
+ params_group = (uid, CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
+ res_group = run_sql(query_group, params_group)
+ if len(res_group):
+ groups = reduce(lambda x, y: x + y, map(lambda x: x[0], res_group))
else:
groups = 0
+
external = count_external_baskets(uid)
+
return (personal, groups, external)
def check_user_owns_baskets(uid, bskids):
@@ -417,7 +346,7 @@ def create_basket(uid, basket_name, topic):
def get_all_items_in_user_personal_baskets(uid,
topic="",
- format='hb'):
+ of='hb'):
"""For the specified user, return all the items in their personal baskets,
grouped by basket if local or as a list if external.
If topic is set, return only that topic's items."""
@@ -425,11 +354,11 @@ def get_all_items_in_user_personal_baskets(uid,
if topic:
topic_clause = """AND ubsk.topic=%s"""
params_local = (uid, uid, topic)
- params_external = (uid, uid, topic, format)
+ params_external = (uid, uid, topic, of)
else:
topic_clause = ""
params_local = (uid, uid)
- params_external = (uid, uid, format)
+ params_external = (uid, uid, of)
query_local = """
SELECT rec.id_bskBASKET,
@@ -525,53 +454,7 @@ def get_all_user_topics(uid):
########################## Actions on baskets #################################
-def get_basket_record(bskid, recid, format='hb'):
- """get record recid in basket bskid
- """
- if recid < 0:
- rec_table = 'bskEXTREC'
- format_table = 'bskEXTFMT'
- id_field = 'id_bskEXTREC'
- sign = '-'
- else:
- rec_table = 'bibrec'
- format_table = 'bibfmt'
- id_field = 'id_bibrec'
- sign = ''
- query = """
- SELECT DATE_FORMAT(record.creation_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
- DATE_FORMAT(record.modification_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
- DATE_FORMAT(bskREC.date_added, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
- user.nickname,
- count(cmt.id_bibrec_or_bskEXTREC),
- DATE_FORMAT(max(cmt.date_creation), '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
- fmt.value
-
- FROM bskREC LEFT JOIN user
- ON bskREC.id_user_who_added_item=user.id
- LEFT JOIN bskRECORDCOMMENT cmt
- ON bskREC.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
- LEFT JOIN %(rec_table)s record
- ON (%(sign)sbskREC.id_bibrec_or_bskEXTREC=record.id)
- LEFT JOIN %(format_table)s fmt
- ON (record.id=fmt.%(id_field)s)
-
- WHERE bskREC.id_bskBASKET=%%s AND
- bskREC.id_bibrec_or_bskEXTREC=%%s AND
- fmt.format=%%s
-
- GROUP BY bskREC.id_bibrec_or_bskEXTREC
- """ % {'rec_table': rec_table,
- 'sign': sign,
- 'format_table': format_table,
- 'id_field':id_field}
- params = (int(bskid), int(recid), format)
- res = run_sql(query, params)
- if res:
- return __decompress_last(res[0])
- return ()
-
-def get_basket_content(bskid, format='hb'):
+def get_basket_content(bskid, of='hb'):
"""Get all records for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
@@ -605,7 +488,7 @@ def get_basket_content(bskid, format='hb'):
ORDER BY rec.score"""
- params = (format, format, int(bskid))
+ params = (of, of, int(bskid))
res = run_sql(query, params)
@@ -615,7 +498,7 @@ def get_basket_content(bskid, format='hb'):
return res
return ()
-def get_basket_item(bskid, recid, format='hb'):
+def get_basket_item(bskid, recid, of='hb'):
"""Get item (recid) for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
@@ -644,7 +527,7 @@ def get_basket_item(bskid, recid, format='hb'):
AND rec.id_bibrec_or_bskEXTREC=%s
GROUP BY rec.id_bibrec_or_bskEXTREC
ORDER BY rec.score"""
- params = (format, format, bskid, recid)
+ params = (of, of, bskid, recid)
res = run_sql(query, params)
if res:
queryU = """UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"""
@@ -1332,7 +1215,7 @@ def get_basket_share_level(bskid):
def get_all_items_in_user_group_baskets(uid,
group=0,
- format='hb'):
+ of='hb'):
"""For the specified user, return all the items in their group baskets,
grouped by basket if local or as a list if external.
If group is set, return only that group's items."""
@@ -1340,11 +1223,11 @@ def get_all_items_in_user_group_baskets(uid,
if group:
group_clause = """AND ugbsk.id_usergroup=%s"""
params_local = (group, uid)
- params_external = (group, uid, format)
+ params_external = (group, uid, of)
else:
group_clause = ""
params_local = (uid,)
- params_external = (uid, format)
+ params_external = (uid, of)
query_local = """
SELECT rec.id_bskBASKET,
@@ -1567,13 +1450,16 @@ def get_all_external_basket_ids_and_names(uid):
def count_external_baskets(uid):
"""Returns the number of external baskets the user is subscribed to."""
+ # TODO: Maybe put this in a try..except ?
+ uid = int(uid)
+
query = """ SELECT COUNT(ubsk.id_bskBASKET)
FROM user_bskBASKET ubsk
LEFT JOIN bskBASKET bsk
ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s)
WHERE bsk.id_owner!=%s"""
- params = (int(uid), int(uid))
+ params = (uid, uid)
res = run_sql(query, params)
@@ -1612,7 +1498,7 @@ def get_all_external_baskets_names(uid,
return run_sql(query, params)
def get_all_items_in_user_public_baskets(uid,
- format='hb'):
+ of='hb'):
"""For the specified user, return all the items in the public baskets they
are subscribed to, grouped by basket if local or as a list if external."""
@@ -1660,7 +1546,7 @@ def get_all_items_in_user_public_baskets(uid,
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
- params_external = (uid, uid, format)
+ params_external = (uid, uid, of)
res_external = run_sql(query_external, params_external)
@@ -1701,7 +1587,7 @@ def get_all_items_in_user_public_baskets_by_matching_notes(uid,
return res
-def get_all_items_in_all_public_baskets(format='hb'):
+def get_all_items_in_all_public_baskets(of='hb'):
"""Return all the items in all the public baskets,
grouped by basket if local or as a list if external."""
@@ -1739,7 +1625,7 @@ def get_all_items_in_all_public_baskets(format='hb'):
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
- params_external = (format,)
+ params_external = (of,)
res_external = run_sql(query_external, params_external)
diff --git a/modules/webbasket/lib/webbasket_templates.py b/modules/webbasket/lib/webbasket_templates.py
index 096d9fffba..6e108bd9e1 100644
--- a/modules/webbasket/lib/webbasket_templates.py
+++ b/modules/webbasket/lib/webbasket_templates.py
@@ -41,9 +41,7 @@
CFG_SITE_RECORD
from invenio.webuser import get_user_info
from invenio.dateutils import convert_datetext_to_dategui
-from invenio.webbasket_dblayer import get_basket_item_title_and_URL, \
- get_basket_ids_and_names
-from invenio.bibformat import format_record
+from invenio.webbasket_dblayer import get_basket_ids_and_names
class Template:
"""Templating class for webbasket module"""
@@ -2116,22 +2114,6 @@ def tmpl_add_group(self, bskid, selected_topic, groups=[], ln=CFG_SITE_LANG):
'submit_label': _("Add group")}
return out
- def tmpl_personal_baskets_selection_box(self,
- baskets=[],
- select_box_name='baskets',
- selected_bskid=None,
- ln=CFG_SITE_LANG):
- """return an HTML popupmenu
- @param baskets: list of (bskid, bsk_name, bsk_topic) tuples
- @param select_box_name: name that will be used for the control
- @param selected_bskid: id of the selcte basket, use None for no selection
- @param ln: language"""
- _ = gettext_set_language(ln)
- elements = [(0, '- ' + _("no basket") + ' -')]
- for (bskid, bsk_name, bsk_topic) in baskets:
- elements.append((bskid, bsk_topic + ' > ' + bsk_name))
- return self.__create_select_menu(select_box_name, elements, selected_bskid)
-
def tmpl_create_guest_warning_box(self, ln=CFG_SITE_LANG):
"""return html warning box for non registered users"""
_ = gettext_set_language(ln)
@@ -3042,7 +3024,7 @@ def tmpl_basket_single_item_content(self,
""" % _("The item you have selected does not exist.")
else:
- (recid, colid, dummy, last_cmt, val, dummy) = item
+ (recid, colid, dummy, dummy, val, dummy) = item
if recid < 0:
external_item_img = ' """ % {'count': index_item,
'icon': external_item_img,
'content': colid >= 0 and val or val and self.tmpl_create_pseudo_item(val) or _("This record does not seem to exist any more"),
- 'notes': notes,
- 'ln': ln}
+ 'notes': notes,}
item_html += """
"""
@@ -4185,9 +4166,9 @@ def tmpl_create_export_as_list(self,
recid)
export_as_html = ""
- for format in list_of_export_as_formats:
+ for of in list_of_export_as_formats:
export_as_html += """%s , """ % \
- (href, format[1], format[0])
+ (href, of[1], of[0])
if export_as_html:
export_as_html = export_as_html[:-2]
out = """
@@ -4201,6 +4182,34 @@ def tmpl_create_export_as_list(self,
return out
+ def tmpl_account_user_baskets(self, personal, group, external, ln = CFG_SITE_LANG):
+ """
+ Information on the user's baskets for the "Your Account" page.
+ """
+
+ _ = gettext_set_language(ln)
+
+ if (personal, group, external) == (0, 0, 0):
+ out = _("You have not created any personal baskets yet, you are not part of any group baskets and you have not subscribed to any public baskets.")
+ else:
+ x_generic_url_open = '' % (CFG_SITE_SECURE_URL, ln)
+ out = _("You have %(x_personal_url_open)s%(x_personal_nb)s personal baskets%(x_personal_url_close)s, you are part of %(x_group_url_open)s%(x_group_nb)s group baskets%(x_group_url_close)s and you are subscribed to %(x_public_url_open)s%(x_public_nb)s public baskets%(x_public_url_close)s.") % \
+ {'x_personal_url_open' : '%s' % (personal > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['PRIVATE'],) or '',),
+ 'x_personal_nb' : str(personal),
+ 'x_personal_url_close' : '%s ' % (personal > 0 and ' ' or '',),
+ 'x_group_url_open' : '%s' % (group > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['GROUP'],) or '',),
+ 'x_group_nb' : str(group),
+ 'x_group_url_close' : '%s ' % (group > 0 and '' or '',),
+ 'x_public_url_open' : '%s' % (external > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['EXTERNAL'],) or '',),
+ 'x_public_nb' : str(external),
+ 'x_public_url_close' : '%s ' % (external > 0 and '' or '',),}
+
+ out += " " + _("You might be interested in looking through %(x_url_open)sall the public baskets%(x_url_close)s.") % \
+ {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln),
+ 'x_url_close' : ' ',}
+
+ return out
+
#############################################
########## SUPPLEMENTARY FUNCTIONS ##########
#############################################
@@ -4349,13 +4358,13 @@ def create_add_box_select_options(category,
if len(personal_basket_list) == 1:
bskids = personal_basket_list[0][1].split(',')
if len(bskids) == 1:
- b = CFG_WEBBASKET_CATEGORIES['PRIVATE'] + '_' + bskids[0]
+ b = CFG_WEBBASKET_CATEGORIES['PRIVATE'] + '_' + bskids[0]
elif len(group_basket_list) == 1:
bskids = group_basket_list[0][1].split(',')
if len(bskids) == 1:
- b = CFG_WEBBASKET_CATEGORIES['GROUP'] + '_' + bskids[0]
+ b = CFG_WEBBASKET_CATEGORIES['GROUP'] + '_' + bskids[0]
- # Create the s and s for the user personal baskets.
+ # Create the optgroups and options for the user personal baskets.
if personal_basket_list:
out += """
""" % ('* ' + _('Your personal baskets') + ' *',)
@@ -4378,7 +4387,7 @@ def create_add_box_select_options(category,
out += """
"""
- # Create the s and s for the user group baskets.
+ # Create the optgroups and options for the user group baskets.
if group_basket_list:
out += """
""" % ('* ' + _('Your group baskets') + ' *',)
diff --git a/modules/webbasket/lib/webbasket_webinterface.py b/modules/webbasket/lib/webbasket_webinterface.py
index f47c6cf01c..13e5d37f60 100644
--- a/modules/webbasket/lib/webbasket_webinterface.py
+++ b/modules/webbasket/lib/webbasket_webinterface.py
@@ -409,7 +409,7 @@ def search(self, req, form):
p=argd['p'],
b=argd['b'],
n=argd['n'],
-# format=argd['of'],
+# record_format=argd['of'],
ln=argd['ln'])
# register event in webstat
diff --git a/modules/webcomment/lib/Makefile.am b/modules/webcomment/lib/Makefile.am
index f8a852e4fe..3f2d60ec9d 100644
--- a/modules/webcomment/lib/Makefile.am
+++ b/modules/webcomment/lib/Makefile.am
@@ -25,7 +25,8 @@ pylib_DATA = webcomment_config.py \
webcommentadminlib.py \
webcomment_regression_tests.py \
webcomment_washer.py \
- webcomment_web_tests.py
+ webcomment_web_tests.py \
+ webcomment_dblayer.py
EXTRA_DIST = $(pylib_DATA)
diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py
index ce92ec10f0..7cbe0fb3c9 100644
--- a/modules/webcomment/lib/webcomment.py
+++ b/modules/webcomment/lib/webcomment.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# This file is part of Invenio.
-# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012 CERN.
+# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -19,6 +19,7 @@
""" Comments and reviews for records """
+
__revision__ = "$Id$"
# non Invenio imports:
@@ -29,33 +30,39 @@
import cgi
import re
from datetime import datetime, timedelta
+from bleach import clean, linkify
# Invenio imports:
from invenio.dbquery import run_sql
-from invenio.config import CFG_PREFIX, \
- CFG_SITE_LANG, \
- CFG_WEBALERT_ALERT_ENGINE_EMAIL,\
- CFG_SITE_SUPPORT_EMAIL,\
- CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,\
- CFG_SITE_URL,\
- CFG_SITE_NAME,\
- CFG_WEBCOMMENT_ALLOW_REVIEWS,\
- CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS,\
- CFG_WEBCOMMENT_ALLOW_COMMENTS,\
- CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL,\
- CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,\
- CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS,\
- CFG_WEBCOMMENT_DEFAULT_MODERATOR, \
- CFG_SITE_RECORD, \
- CFG_WEBCOMMENT_EMAIL_REPLIES_TO, \
- CFG_WEBCOMMENT_ROUND_DATAFIELD, \
- CFG_WEBCOMMENT_RESTRICTION_DATAFIELD, \
- CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH
-from invenio.webmessage_mailutils import \
- email_quote_txt, \
- email_quoted_txt2html
-from invenio.htmlutils import tidy_html
+from invenio.config import \
+ CFG_PREFIX, \
+ CFG_SITE_LANG, \
+ CFG_WEBALERT_ALERT_ENGINE_EMAIL,\
+ CFG_SITE_SUPPORT_EMAIL,\
+ CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,\
+ CFG_SITE_URL,\
+ CFG_SITE_SECURE_URL,\
+ CFG_SITE_NAME,\
+ CFG_WEBCOMMENT_ALLOW_REVIEWS,\
+ CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS,\
+ CFG_WEBCOMMENT_ALLOW_COMMENTS,\
+ CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL,\
+ CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,\
+ CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS,\
+ CFG_WEBCOMMENT_DEFAULT_MODERATOR, \
+ CFG_SITE_RECORD, \
+ CFG_WEBCOMMENT_EMAIL_REPLIES_TO, \
+ CFG_WEBCOMMENT_ROUND_DATAFIELD, \
+ CFG_WEBCOMMENT_RESTRICTION_DATAFIELD, \
+ CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH, \
+ CFG_WEBCOMMENT_ENABLE_HTML_EMAILS, \
+ CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR, \
+ CFG_WEBCOMMENT_ENABLE_MARKDOWN_TEXT_RENDERING
+from invenio.webmessage_mailutils import email_quote_txt
+from invenio.htmlutils import \
+ CFG_HTML_BUFFER_ALLOWED_TAG_WHITELIST, \
+ CFG_HTML_BUFFER_ALLOWED_ATTRIBUTE_WHITELIST
from invenio.webuser import get_user_info, get_email, collect_user_info
from invenio.dateutils import convert_datetext_to_dategui, \
datetext_default, \
@@ -64,9 +71,10 @@
from invenio.errorlib import register_exception
from invenio.messages import wash_language, gettext_set_language
from invenio.urlutils import wash_url_argument
-from invenio.webcomment_config import CFG_WEBCOMMENT_ACTION_CODE, \
- InvenioWebCommentError, \
- InvenioWebCommentWarning
+from invenio.webcomment_config import \
+ CFG_WEBCOMMENT_ACTION_CODE, \
+ CFG_WEBCOMMENT_BODY_FORMATS, \
+ CFG_WEBCOMMENT_OUTPUT_FORMATS
from invenio.access_control_engine import acc_authorize_action
from invenio.search_engine import \
guess_primary_collection_of_a_record, \
@@ -74,7 +82,9 @@
get_collection_reclist, \
get_colID
from invenio.search_engine_utils import get_fieldvalues
-from invenio.webcomment_washer import EmailWasher
+from invenio.webcomment_dblayer import \
+ set_comment_to_bibdoc_relation,\
+ get_comment_to_bibdoc_relations
try:
import invenio.template
webcomment_templates = invenio.template.load('webcomment')
@@ -82,7 +92,28 @@
pass
-def perform_request_display_comments_or_remarks(req, recID, display_order='od', display_since='all', nb_per_page=100, page=1, ln=CFG_SITE_LANG, voted=-1, reported=-1, subscribed=0, reviews=0, uid=-1, can_send_comments=False, can_attach_files=False, user_is_subscribed_to_discussion=False, user_can_unsubscribe_from_discussion=False, display_comment_rounds=None):
+def perform_request_display_comments_or_remarks(
+ req,
+ recID,
+ display_order='od',
+ display_since='all',
+ nb_per_page=100,
+ page=1,
+ ln=CFG_SITE_LANG,
+ voted=-1,
+ reported=-1,
+ subscribed=0,
+ reviews=0,
+ uid=-1,
+ can_send_comments=False,
+ can_attach_files=False,
+ user_is_subscribed_to_discussion=False,
+ user_can_unsubscribe_from_discussion=False,
+ display_comment_rounds=None,
+ filter_for_text=None,
+ filter_for_file=None,
+ relate_to_file=None
+):
"""
Returns all the comments (reviews) of a specific internal record or external basket record.
@param recID: record id where (internal record IDs > 0) or (external basket record IDs < -100)
@@ -158,43 +189,24 @@ def perform_request_display_comments_or_remarks(req, recID, display_order='od',
nb_reviews = get_nb_reviews(recID, count_deleted=False)
nb_comments = get_nb_comments(recID, count_deleted=False)
+ res_related_files = get_comment_to_bibdoc_relations(recID)
+ related_files = {}
+ if res_related_files:
+ for related_file in res_related_files:
+ related_files[related_file['id_comment']] = related_file
# checking non vital arguemnts - will be set to default if wrong
#if page <= 0 or page.lower() != 'all':
if page < 0:
page = 1
- try:
- raise InvenioWebCommentWarning(_('Bad page number --> showing first page.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_INVALID_PAGE_NB',))
if nb_per_page < 0:
nb_per_page = 100
- try:
- raise InvenioWebCommentWarning(_('Bad number of results per page --> showing 10 results per page.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_INVALID_NB_RESULTS_PER_PAGE',))
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
if display_order not in ['od', 'nd', 'hh', 'lh', 'hs', 'ls']:
display_order = 'hh'
- try:
- raise InvenioWebCommentWarning(_('Bad display order --> showing most helpful first.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_INVALID_REVIEW_DISPLAY_ORDER',))
else:
if display_order not in ['od', 'nd']:
display_order = 'od'
- try:
- raise InvenioWebCommentWarning(_('Bad display order --> showing oldest first.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_INVALID_DISPLAY_ORDER',))
if not display_comment_rounds:
display_comment_rounds = []
@@ -207,12 +219,6 @@ def perform_request_display_comments_or_remarks(req, recID, display_order='od',
last_page = 1
if page > last_page:
page = 1
- try:
- raise InvenioWebCommentWarning(_('Bad page number --> showing first page.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(("WRN_WEBCOMMENT_INVALID_PAGE_NB",))
if nb_res > nb_per_page: # if more than one page of results
if page < last_page:
res = res[(page-1)*(nb_per_page) : (page*nb_per_page)]
@@ -228,69 +234,42 @@ def perform_request_display_comments_or_remarks(req, recID, display_order='od',
if reviews:
res = [row[:] + (row[10] in user_collapsed_comments,) for row in res]
else:
- res = [row[:] + (row[6] in user_collapsed_comments,) for row in res]
+ res = [row[:] + (row[7] in user_collapsed_comments,) for row in res]
# Send to template
avg_score = 0.0
- if not CFG_WEBCOMMENT_ALLOW_COMMENTS and not CFG_WEBCOMMENT_ALLOW_REVIEWS: # comments not allowed by admin
- try:
- raise InvenioWebCommentError(_('Comments on records have been disallowed by the administrator.'))
- except InvenioWebCommentError, exc:
- register_exception(req=req)
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- # errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
+ # comments not allowed by admin
+ if not CFG_WEBCOMMENT_ALLOW_COMMENTS and not CFG_WEBCOMMENT_ALLOW_REVIEWS:
+ body = webcomment_templates.tmpl_error(
+ _('Comments on records have been disallowed by the'
+ ' administrator.'), ln)
+ return body
if reported > 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback has been recorded, many thanks.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED',))
+ warnings.append((_('Your feedback has been recorded, many thanks.'),
+ 'green'))
elif reported == 0:
- try:
- raise InvenioWebCommentWarning(_('You have already reported an abuse for this comment.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_ALREADY_REPORTED',))
+ warnings.append((_('You have already reported an abuse for this'
+ ' comment.'), ''))
elif reported == -2:
- try:
- raise InvenioWebCommentWarning(_('The comment you have reported no longer exists.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_INVALID_REPORT',))
+ warnings.append((_('The comment you have reported no longer '
+ 'exists.'), ''))
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
avg_score = calculate_avg_score(res)
if voted > 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback has been recorded, many thanks.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED',))
+ warnings.append((_('Your feedback has been recorded, many'
+ ' thanks.'), 'green'))
elif voted == 0:
- try:
- raise InvenioWebCommentWarning(_('Sorry, you have already voted. This vote has not been recorded.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_ALREADY_VOTED',))
+ warnings.append((_('Sorry, you have already voted. This vote has '
+ 'not been recorded.'), ''))
if subscribed == 1:
- try:
- raise InvenioWebCommentWarning(_('You have been subscribed to this discussion. From now on, you will receive an email whenever a new comment is posted.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_SUBSCRIBED',))
+ warnings.append(
+ (_('You have been subscribed to this discussion. From now on, you'
+ ' will receive an email whenever a new comment is posted.'),
+ 'green')
+ )
elif subscribed == -1:
- try:
- raise InvenioWebCommentWarning(_('You have been unsubscribed from this discussion.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_UNSUBSCRIBED',))
+ warnings.append((_('You have been unsubscribed from this discussion.'),
+ 'green'))
grouped_comments = group_comments_by_round(res, reviews)
@@ -304,25 +283,34 @@ def perform_request_display_comments_or_remarks(req, recID, display_order='od',
display_comment_rounds.append(grouped_comments[-1][0])
display_comment_rounds.remove('latest')
- body = webcomment_templates.tmpl_get_comments(req,
- recID,
- ln,
- nb_per_page, page, last_page,
- display_order, display_since,
- CFG_WEBCOMMENT_ALLOW_REVIEWS,
- grouped_comments, nb_comments, avg_score,
- warnings,
- border=0,
- reviews=reviews,
- total_nb_reviews=nb_reviews,
- uid=uid,
- can_send_comments=can_send_comments,
- can_attach_files=can_attach_files,
- user_is_subscribed_to_discussion=\
- user_is_subscribed_to_discussion,
- user_can_unsubscribe_from_discussion=\
- user_can_unsubscribe_from_discussion,
- display_comment_rounds=display_comment_rounds)
+ body = webcomment_templates.tmpl_get_comments(
+ req,
+ recID,
+ ln,
+ nb_per_page,
+ page,
+ last_page,
+ display_order,
+ display_since,
+ CFG_WEBCOMMENT_ALLOW_REVIEWS,
+ grouped_comments,
+ nb_comments,
+ avg_score,
+ warnings,
+ border=0,
+ reviews=reviews,
+ total_nb_reviews=nb_reviews,
+ uid=uid,
+ can_send_comments=can_send_comments,
+ can_attach_files=can_attach_files,
+ user_is_subscribed_to_discussion=user_is_subscribed_to_discussion,
+ user_can_unsubscribe_from_discussion=user_can_unsubscribe_from_discussion,
+ display_comment_rounds=display_comment_rounds,
+ filter_for_text=filter_for_text,
+ filter_for_file=filter_for_file,
+ relate_to_file=relate_to_file,
+ related_files=related_files
+ )
return body
def perform_request_vote(cmt_id, client_ip_address, value, uid=-1):
@@ -469,21 +457,23 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1):
params = (cmt_id, uid, client_ip_address, action_date, action_code)
run_sql(query, params)
if nb_abuse_reports % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN == 0:
- (cmt_id2,
+ (dummy,
id_bibrec,
id_user,
cmt_body,
+ cmt_body_format,
cmt_date,
cmt_star,
- cmt_vote, cmt_nb_votes_total,
+ dummy,
+ dummy,
cmt_title,
cmt_reported,
- round_name,
- restriction) = query_get_comment(cmt_id)
+ dummy,
+ dummy) = query_get_comment(cmt_id)
(user_nb_abuse_reports,
user_votes,
user_nb_votes_total) = query_get_user_reports_and_votes(int(id_user))
- (nickname, user_email, last_login) = query_get_user_contact_info(id_user)
+ (nickname, user_email, dummy) = query_get_user_contact_info(id_user)
from_addr = '%s Alert Engine <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(cmt_id)
to_addrs = get_collection_moderators(comment_collection)
@@ -504,16 +494,16 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1):
%(review_stuff)s
body =
---start body---
+
%(cmt_body)s
+
---end body---
Please go to the record page %(comment_admin_link)s to delete this message if necessary. A warning will be sent to the user in question.''' % \
- { 'cfg-report_max' : CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,
- 'nickname' : nickname,
+ { 'nickname' : nickname,
'user_email' : user_email,
'uid' : id_user,
'user_nb_abuse_reports' : user_nb_abuse_reports,
- 'user_votes' : user_votes,
'votes' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"total number of positive votes\t= %s\n\t\ttotal number of negative votes\t= %s" % \
(user_votes, (user_nb_votes_total - user_votes)) or "\n",
@@ -523,7 +513,7 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1):
'cmt_reported' : cmt_reported,
'review_stuff' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"star score\t= %s\n\treview title\t= %s" % (cmt_star, cmt_title) or "",
- 'cmt_body' : cmt_body,
+ 'cmt_body' : webcomment_templates.tmpl_prepare_comment_body(cmt_body, cmt_body_format, CFG_WEBCOMMENT_OUTPUT_FORMATS["TEXT"]["EMAIL"]),
'comment_admin_link' : CFG_SITE_URL + "/"+ CFG_SITE_RECORD +"/" + str(id_bibrec) + '/comments#' + str(cmt_id),
'user_admin_link' : "user_admin_link" #! FIXME
}
@@ -531,7 +521,72 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1):
#FIXME to be added to email when websession module is over:
#If you wish to ban the user, you can do so via the User Admin Panel %(user_admin_link)s.
- send_email(from_addr, to_addrs, subject, body)
+ if CFG_WEBCOMMENT_ENABLE_HTML_EMAILS:
+ html_content = """
+The following comment has been reported a total of %(cmt_reported)s times.
+
+Author:
+
+ nickname = %(nickname)s
+ email = <%(user_email)s >
+ user_id = %(uid)s
+ This user has:
+
+ total number of reports = %(user_nb_abuse_reports)s
+ %(votes)s
+
+
+
+Comment:
+
+ comment_id = %(cmt_id)s
+ record_id = %(id_bibrec)s
+ date written = %(cmt_date)s
+ nb reports = %(cmt_reported)s
+ %(review_stuff)s
+ body =
+
+
+<--------------->Please go to the record page <%(comment_admin_link)s > to delete this message if necessary.
+ A warning will be sent to the user in question.
""" % {
+ 'cfg-report_max' : CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,
+ 'nickname' : cgi.escape(nickname),
+ 'user_email' : user_email,
+ 'uid' : id_user,
+ 'user_nb_abuse_reports' : user_nb_abuse_reports,
+ 'user_votes' : user_votes,
+ 'votes' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
+ "total number of positive votes = %s \ntotal number of negative votes= %s " % \
+ (user_votes, (user_nb_votes_total - user_votes))
+ or "",
+ 'cmt_id' : cmt_id,
+ 'id_bibrec' : id_bibrec,
+ 'cmt_date' : cmt_date,
+ 'cmt_reported' : cmt_reported,
+ 'review_stuff' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
+ "star score = %s \nreview title = %s " % (cmt_star, cmt_title)
+ or "",
+ 'cmt_body' : webcomment_templates.tmpl_prepare_comment_body(cmt_body, cmt_body_format, CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["EMAIL"]),
+ 'comment_admin_link' : CFG_SITE_URL + "/"+ CFG_SITE_RECORD +"/" + str(id_bibrec) + '/comments#' + str(cmt_id),
+ 'user_admin_link' : "user_admin_link", #! FIXME
+}
+ else:
+ html_content = None
+
+ return int(send_email(
+ fromaddr=from_addr,
+ toaddr=to_addrs,
+ subject=subject,
+ content=body,
+ html_content=html_content
+ ))
+
return 1
def check_user_can_report(cmt_id, client_ip_address, uid=-1):
@@ -607,6 +662,7 @@ def query_get_comment(comID):
id_bibrec,
id_user,
body,
+ body_format,
DATE_FORMAT(date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
star_score,
nb_votes_yes,
@@ -724,17 +780,19 @@ def query_retrieve_comments_or_remarks(recID, display_order='od', display_since=
cmt.body,
cmt.status,
cmt.nb_abuse_reports,
- %(ranking)s cmt.id,
+ %(ranking)s cmt.title,
+ cmt.id,
cmt.round_name,
cmt.restriction,
- %(reply_to_column)s
+ %(reply_to_column)s,
+ cmt.body_format
FROM cmtRECORDCOMMENT cmt LEFT JOIN user ON
user.id=cmt.id_user
WHERE cmt.id_bibrec=%%s
%(ranking_only)s
%(display_since)s
ORDER BY %(display_order)s
- """ % {'ranking' : ranking and ' cmt.nb_votes_yes, cmt.nb_votes_total, cmt.star_score, cmt.title, ' or '',
+ """ % {'ranking' : ranking and ' cmt.nb_votes_yes, cmt.nb_votes_total, cmt.star_score, ' or '',
'ranking_only' : ranking and ' AND cmt.star_score>0 ' or ' AND cmt.star_score=0 ',
# 'id_bibrec' : recID > 0 and 'cmt.id_bibrec' or 'cmt.id_bibrec_or_bskEXTREC',
# 'table' : recID > 0 and 'cmtRECORDCOMMENT' or 'bskRECORDCOMMENT',
@@ -753,7 +811,7 @@ def query_retrieve_comments_or_remarks(recID, display_order='od', display_since=
restriction = row[12]
else:
# when dealing with comments, row[8] holds restriction info:
- restriction = row[8]
+ restriction = row[9]
if user_info and check_user_can_view_comment(user_info, None, restriction)[0] != 0:
# User cannot view comment. Look further
continue
@@ -849,10 +907,21 @@ def get_reply_order_cache_data(comid):
return "%s%s%s%s" % (chr((comid >> 24) % 256), chr((comid >> 16) % 256),
chr((comid >> 8) % 256), chr(comid % 256))
-def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
- note="", score=0, priority=0,
- client_ip_address='', editor_type='textarea',
- req=None, reply_to=None, attached_files=None):
+def query_add_comment_or_remark(
+ reviews=0,
+ recID=0,
+ uid=-1,
+ msg="",
+ note="",
+ score=0,
+ priority=0,
+ client_ip_address='',
+ editor_type='textarea',
+ req=None,
+ reply_to=None,
+ attached_files=None,
+ relate_file=None
+):
"""
Private function
Insert a comment/review or remarkinto the database
@@ -865,19 +934,24 @@ def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
@param editor_type: the kind of editor used to submit the comment: 'textarea', 'ckeditor'
@param req: request object. If provided, email notification are sent after we reply to user request.
@param reply_to: the id of the comment we are replying to with this inserted comment.
+ @param relate_file: dictionary containing all the information about the
+ relation of the comment being submitted to the bibdocfile that was
+ chosen. ("id_bibdoc:version:mime")
@return: integer >0 representing id if successful, integer 0 if not
"""
+
current_date = calculate_start_date('0d')
- #change utf-8 message into general unicode
- msg = msg.decode('utf-8')
- note = note.decode('utf-8')
- #change general unicode back to utf-8
- msg = msg.encode('utf-8')
- note = note.encode('utf-8')
- msg_original = msg
+
+ # NOTE: Change utf-8 message into general unicode and back to utf-8
+ # (Why do we do this here?)
+ msg = msg.decode('utf-8').encode('utf-8')
+ note = note.decode('utf-8').encode('utf-8')
+
(restriction, round_name) = get_record_status(recID)
+
if attached_files is None:
attached_files = {}
+
if reply_to and CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH >= 0:
# Check that we have not reached max depth
comment_ancestors = get_comment_ancestors(reply_to)
@@ -889,48 +963,32 @@ def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
# Inherit restriction and group/round of 'parent'
comment = query_get_comment(reply_to)
if comment:
- (round_name, restriction) = comment[10:12]
- if editor_type == 'ckeditor':
- # Here we remove the line feeds introduced by CKEditor (they
- # have no meaning for the user) and replace the HTML line
- # breaks by linefeeds, so that we are close to an input that
- # would be done without the CKEditor. That's much better if a
- # reply to a comment is made with a browser that does not
- # support CKEditor.
- msg = msg.replace('\n', '').replace('\r', '')
-
- # We clean the quotes that could have been introduced by
- # CKEditor when clicking the 'quote' button, as well as those
- # that we have introduced when quoting the original message.
- # We can however not use directly '>>' chars to quote, as it
- # will be washed/fixed when calling tidy_html(): double-escape
- # all > first, and use >>
- msg = msg.replace('>', '>')
- msg = re.sub('^\s*\s*<(p|div).*?>', '>>', msg)
- msg = re.sub('\s*', '', msg)
- # Then definitely remove any blockquote, whatever it is
- msg = re.sub('', '', msg)
- msg = re.sub('', '
', msg)
- # Tidy up the HTML
- msg = tidy_html(msg)
- # We remove EOL that might have been introduced when tidying
- msg = msg.replace('\n', '').replace('\r', '')
- # Now that HTML has been cleaned, unescape >
- msg = msg.replace('>', '>')
- msg = msg.replace('>', '>')
- msg = re.sub(' or
got inserted, interpret
- # these as new lines (with a sad trick to do it only once)
- # (note that it has been deactivated, as it is messing up
- # indentation with >>)
- #msg = msg.replace('
<', '\n<')
- #msg = msg.replace('<', '\n<')
+ (round_name, restriction) = comment[11:13]
+
+ if editor_type == "ckeditor":
+ msg = linkify(msg)
+ msg = clean(
+ msg,
+ tags=CFG_HTML_BUFFER_ALLOWED_TAG_WHITELIST,
+ attributes=CFG_HTML_BUFFER_ALLOWED_ATTRIBUTE_WHITELIST,
+ strip=True
+ )
+ body_format = CFG_WEBCOMMENT_BODY_FORMATS["HTML"]
+
+ elif editor_type == "textarea":
+ if CFG_WEBCOMMENT_ENABLE_MARKDOWN_TEXT_RENDERING:
+ body_format = CFG_WEBCOMMENT_BODY_FORMATS["MARKDOWN"]
+ else:
+ body_format = CFG_WEBCOMMENT_BODY_FORMATS["TEXT"]
+
+ else:
+ # NOTE: it should really be one of the above 2 types.
+ body_format = CFG_WEBCOMMENT_BODY_FORMATS["TEXT"]
query = """INSERT INTO cmtRECORDCOMMENT (id_bibrec,
id_user,
body,
+ body_format,
date_creation,
star_score,
nb_votes_total,
@@ -938,12 +996,23 @@ def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
round_name,
restriction,
in_reply_to_id_cmtRECORDCOMMENT)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
- params = (recID, uid, msg, current_date, score, 0, note, round_name, restriction, reply_to or 0)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
+ params = (recID, uid, msg, body_format, current_date, score, 0, note, round_name, restriction, reply_to or 0)
res = run_sql(query, params)
if res:
new_comid = int(res)
move_attached_files_to_storage(attached_files, recID, new_comid)
+
+ # Relate comment with a bibdocfile if one was chosen
+ if relate_file and len(relate_file.split(":")) >= 2:
+ id_bibdoc, doc_version = relate_file.split(":")[:2]
+ set_comment_to_bibdoc_relation(
+ recID,
+ new_comid,
+ id_bibdoc,
+ doc_version
+ )
+
parent_reply_order = run_sql("""SELECT reply_order_cached_data from cmtRECORDCOMMENT where id=%s""", (reply_to,))
if not parent_reply_order or parent_reply_order[0][0] is None:
# This is not a reply, but a first 0-level comment
@@ -968,7 +1037,7 @@ def notify_subscribers_callback(data):
@param data: contains the necessary parameters in a tuple:
(recid, uid, comid, msg, note, score, editor_type, reviews)
"""
- recid, uid, comid, msg, note, score, editor_type, reviews = data
+ recid, uid, comid, msg, body_format, note, score, editor_type, reviews = data
# Email this comment to 'subscribers'
(subscribers_emails1, subscribers_emails2) = \
get_users_subscribed_to_discussion(recid)
@@ -976,12 +1045,13 @@ def notify_subscribers_callback(data):
emails1=subscribers_emails1,
emails2=subscribers_emails2,
comID=comid, msg=msg,
+ body_format=body_format,
note=note, score=score,
editor_type=editor_type, uid=uid)
# Register our callback to notify subscribed people after
# having replied to our current user.
- data = (recID, uid, res, msg, note, score, editor_type, reviews)
+ data = (recID, uid, res, msg, body_format, note, score, editor_type, reviews)
if req:
req.register_cleanup(notify_subscribers_callback, data)
else:
@@ -1116,7 +1186,7 @@ def get_users_subscribed_to_discussion(recID, check_authorizations=True):
uid = row[0]
if check_authorizations:
user_info = collect_user_info(uid)
- (auth_code, auth_msg) = check_user_can_view_comments(user_info, recID)
+ (auth_code, dummy) = check_user_can_view_comments(user_info, recID)
else:
# Don't check and grant access
auth_code = False
@@ -1152,6 +1222,7 @@ def get_users_subscribed_to_discussion(recID, check_authorizations=True):
def email_subscribers_about_new_comment(recID, reviews, emails1,
emails2, comID, msg="",
+ body_format=CFG_WEBCOMMENT_BODY_FORMATS["HTML"],
note="", score=0,
editor_type='textarea',
ln=CFG_SITE_LANG, uid=-1):
@@ -1170,11 +1241,11 @@ def email_subscribers_about_new_comment(recID, reviews, emails1,
@rtype: bool
@return: True if email was sent okay, False if it was not.
"""
+
_ = gettext_set_language(ln)
if not emails1 and not emails2:
return 0
-
# Get title
titles = get_fieldvalues(recID, "245__a")
if not titles:
@@ -1204,67 +1275,101 @@ def email_subscribers_about_new_comment(recID, reviews, emails1,
{'report_number': report_numbers and ('[' + report_numbers[0] + '] ') or '',
'title': title}
- washer = EmailWasher()
- msg = washer.wash(msg)
- msg = msg.replace('>>', '>')
- email_content = msg
if note:
- email_content = note + email_content
-
- # Send emails to people who can unsubscribe
- email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
- title,
- reviews,
- comID,
- report_numbers,
- can_unsubscribe=True,
- ln=ln,
- uid=uid)
-
- email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
- title,
- reviews,
- comID,
- report_numbers,
- can_unsubscribe=True,
- ln=ln)
+ email_content = note + msg
+ else:
+ email_content = msg
+
+ def send_email_kwargs(
+ recID=recID,
+ title=title,
+ reviews=reviews,
+ comID=comID,
+ report_numbers=report_numbers,
+ can_unsubscribe=True,
+ uid=uid,
+ ln=ln,
+ body_format=body_format,
+ fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
+ toaddr=[],
+ subject=email_subject,
+ content=email_content
+ ):
+
+ tmpl_email_new_comment_footer_kwargs = {
+ "recID" : recID,
+ "title" : title,
+ "reviews" : reviews,
+ "comID" : comID,
+ "report_numbers" : report_numbers,
+ "can_unsubscribe" : can_unsubscribe,
+ "ln" : ln,
+ "html_p" : False,
+ }
+
+ tmpl_email_new_comment_header_kwargs = tmpl_email_new_comment_footer_kwargs.copy()
+ tmpl_email_new_comment_header_kwargs.update({
+ "uid" : uid,
+ })
+
+ kwargs = {
+ "fromaddr" : fromaddr,
+ "toaddr" : toaddr,
+ "subject" : subject,
+ "content" : webcomment_templates.tmpl_prepare_comment_body(
+ content,
+ body_format,
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["TEXT"]["EMAIL"]
+ ),
+ "header" : webcomment_templates.tmpl_email_new_comment_header(**tmpl_email_new_comment_header_kwargs),
+ "footer" : webcomment_templates.tmpl_email_new_comment_footer(**tmpl_email_new_comment_footer_kwargs),
+ "html_content" : "",
+ "html_header" : None,
+ "html_footer" : None,
+ "ln" : ln,
+ }
+
+ if CFG_WEBCOMMENT_ENABLE_HTML_EMAILS:
+ tmpl_email_new_comment_footer_kwargs.update({
+ "html_p" : True,
+ })
+ tmpl_email_new_comment_header_kwargs.update({
+ "html_p" : True,
+ })
+ kwargs.update({
+ "html_content" : webcomment_templates.tmpl_prepare_comment_body(
+ content,
+ body_format,
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["EMAIL"]
+ ),
+ "html_header" : webcomment_templates.tmpl_email_new_comment_header(**tmpl_email_new_comment_header_kwargs),
+ "html_footer" : webcomment_templates.tmpl_email_new_comment_footer(**tmpl_email_new_comment_footer_kwargs),
+ })
+
+ return kwargs
+
+ # First, send emails to people who can unsubscribe.
res1 = True
if emails1:
- res1 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
- toaddr=emails1,
- subject=email_subject,
- content=email_content,
- header=email_header,
- footer=email_footer,
- ln=ln)
-
- # Then send email to people who have been automatically
- # subscribed to the discussion (they cannot unsubscribe)
- email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
- title,
- reviews,
- comID,
- report_numbers,
- can_unsubscribe=False,
- ln=ln,
- uid=uid)
-
- email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
- title,
- reviews,
- comID,
- report_numbers,
- can_unsubscribe=False,
- ln=ln)
+ res1 = send_email(
+ **send_email_kwargs(
+ can_unsubscribe=True,
+ body_format=body_format,
+ toaddr=emails1
+ )
+ )
+
+ # Then, send emails to people who have been automatically subscribed
+ # to the discussion and cannot unsubscribe.
res2 = True
if emails2:
- res2 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
- toaddr=emails2,
- subject=email_subject,
- content=email_content,
- header=email_header,
- footer=email_footer,
- ln=ln)
+ res2 = send_email(
+ **send_email_kwargs(
+ can_unsubscribe=False,
+ body_format=body_format,
+ toaddr=emails2
+ )
+ )
return res1 and res2
@@ -1419,53 +1524,35 @@ def get_first_comments_or_remarks(recID=-1,
first_res_comments = res_comments[:nb_comments]
else:
first_res_comments = res_comments
- else: #error
- try:
- raise InvenioWebCommentError(_('%s is an invalid record ID') % recID)
- except InvenioWebCommentError, exc:
- register_exception()
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_RECID_INVALID', recID)) #!FIXME dont return error anywhere since search page
+ else:
+ body = webcomment_templates.tmpl_error(
+ _('%s is an invalid record ID') % recID, ln)
+ return body
# comment
if recID >= 1:
comments = reviews = ""
if reported > 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback has been recorded, many thanks.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED_GREEN_TEXT',))
+ warnings.append((_('Your feedback has been recorded, many '
+ 'thanks.'), 'green'))
elif reported == 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback could not be recorded, please try again.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_NOT_RECORDED_RED_TEXT',))
- if CFG_WEBCOMMENT_ALLOW_COMMENTS: # normal comments
+ warnings.append((_('Your feedback could not be recorded, please'
+ ' try again.'), ''))
+ # normal comments
+ if CFG_WEBCOMMENT_ALLOW_COMMENTS:
grouped_comments = group_comments_by_round(first_res_comments, ranking=0)
comments = webcomment_templates.tmpl_get_first_comments_without_ranking(recID, ln, grouped_comments, nb_res_comments, warnings)
if show_reviews:
- if CFG_WEBCOMMENT_ALLOW_REVIEWS: # ranked comments
- #calculate average score
+ # ranked comments
+ if CFG_WEBCOMMENT_ALLOW_REVIEWS:
+ # calculate average score
avg_score = calculate_avg_score(res_reviews)
if voted > 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback has been recorded, many thanks.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, 'green'))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED_GREEN_TEXT',))
+ warnings.append((_('Your feedback has been recorded, '
+ 'many thanks.'), 'green'))
elif voted == 0:
- try:
- raise InvenioWebCommentWarning(_('Your feedback could not be recorded, please try again.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_FEEDBACK_NOT_RECORDED_RED_TEXT',))
+ warnings.append((_('Your feedback could not be recorded, '
+ 'please try again.'), ''))
grouped_reviews = group_comments_by_round(first_res_reviews, ranking=0)
reviews = webcomment_templates.tmpl_get_first_comments_with_ranking(recID, ln, grouped_reviews, nb_res_reviews, avg_score, warnings)
return (comments, reviews)
@@ -1480,7 +1567,7 @@ def group_comments_by_round(comments, ranking=0):
comment_rounds = {}
ordered_comment_round_names = []
for comment in comments:
- comment_round_name = ranking and comment[11] or comment[7]
+ comment_round_name = ranking and comment[11] or comment[8]
if not comment_rounds.has_key(comment_round_name):
comment_rounds[comment_round_name] = []
ordered_comment_round_names.append(comment_round_name)
@@ -1516,23 +1603,26 @@ def calculate_avg_score(res):
avg_score = 5.0
return avg_score
-def perform_request_add_comment_or_remark(recID=0,
- uid=-1,
- action='DISPLAY',
- ln=CFG_SITE_LANG,
- msg=None,
- score=None,
- note=None,
- priority=None,
- reviews=0,
- comID=0,
- client_ip_address=None,
- editor_type='textarea',
- can_attach_files=False,
- subscribe=False,
- req=None,
- attached_files=None,
- warnings=None):
+def perform_request_add_comment_or_remark(
+ recID=0,
+ uid=-1,
+ action='DISPLAY',
+ ln=CFG_SITE_LANG,
+ msg=None,
+ score=None,
+ note=None,
+ priority=None,
+ reviews=0,
+ comID=0,
+ client_ip_address=None,
+ editor_type='textarea',
+ can_attach_files=False,
+ subscribe=False,
+ req=None,
+ attached_files=None,
+ warnings=None,
+ relate_file=None
+):
"""
Add a comment/review or remark
@param recID: record id
@@ -1558,24 +1648,20 @@ def perform_request_add_comment_or_remark(recID=0,
- html add form if action is display or reply
- html successful added form if action is submit
"""
+
_ = gettext_set_language(ln)
+
if warnings is None:
warnings = []
- actions = ['DISPLAY', 'REPLY', 'SUBMIT']
_ = gettext_set_language(ln)
## check arguments
check_recID_is_in_range(recID, warnings, ln)
if uid <= 0:
- try:
- raise InvenioWebCommentError(_('%s is an invalid user ID.') % uid)
- except InvenioWebCommentError, exc:
- register_exception()
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_UID_INVALID', uid))
- return ''
+ body = webcomment_templates.tmpl_error(
+ _('%s is an invalid user ID.') % uid, ln)
+ return body
if attached_files is None:
attached_files = {}
@@ -1590,26 +1676,28 @@ def perform_request_add_comment_or_remark(recID=0,
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
- return webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files)
+ return webcomment_templates.tmpl_add_comment_form(
+ recID,
+ uid,
+ nickname,
+ ln,
+ msg,
+ warnings,
+ can_attach_files=can_attach_files,
+ relate_to_file=relate_file
+ )
else:
- try:
- raise InvenioWebCommentError(_('Comments on records have been disallowed by the administrator.'))
- except InvenioWebCommentError, exc:
- register_exception(req=req)
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
+ body = webcomment_templates.tmpl_error(
+ _('Comments on records have been disallowed by the '
+ 'administrator.'), ln)
+ return body
elif action == 'REPLY':
+
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
- try:
- raise InvenioWebCommentError(_('Cannot reply to a review.'))
- except InvenioWebCommentError, exc:
- register_exception(req=req)
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_REPLY_REVIEW',))
- return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
+ body = webcomment_templates.tmpl_error(
+ _('Cannot reply to a review.'), ln)
+ return body
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
textual_msg = msg
if comID > 0:
@@ -1617,132 +1705,165 @@ def perform_request_add_comment_or_remark(recID=0,
if comment:
user_info = get_user_info(comment[2])
if user_info:
- date_creation = convert_datetext_to_dategui(str(comment[4]))
- # Build two msg: one mostly textual, the other one with HTML markup, for the CkEditor.
- msg = _("%(x_name)s wrote on %(x_date)s:")% {'x_name': user_info[2], 'x_date': date_creation}
- textual_msg = msg
- # 1 For CkEditor input
- msg += '\n\n'
- msg += comment[3]
- msg = email_quote_txt(text=msg)
- # Now that we have a text-quoted version, transform into
- # something that CkEditor likes, using that
- # do still enable users to insert comments inline
- msg = email_quoted_txt2html(text=msg,
- indent_html=('', '
%s
+
+ %s
+
+
+ """ % (
+ user_wrote_on,
+ webcomment_templates.tmpl_prepare_comment_body(
+ comment[3],
+ comment[4],
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["CKEDITOR"]
+ )
+ )
+
+ textual_msg = "%s\n\n%s\n\n" % (
+ user_wrote_on,
+ email_quote_txt(
+ webcomment_templates.tmpl_prepare_comment_body(
+ comment[3],
+ comment[4],
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["TEXT"]["TEXTAREA"]
+ ),
+ # TODO: Maybe always use a single ">" for quotations?
+ indent_txt=">",
+ # If we are using CKEditor, then we need to escape
+ # the textual message before passing it to the
+ # editor. "email_quote_txt" can do that for us.
+ # Normally, we could check the "editor_type"
+ # argument that was passed to this function.
+ # However, it is usually an empty string since
+ # it is comming from the "add" interface.
+ # Instead let's directly use the config variable.
+ #escape_p=(editor_type == "ckeditor")
+ escape_p=CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR
+ )
+ )
+
+ return webcomment_templates.tmpl_add_comment_form(
+ recID,
+ uid,
+ nickname,
+ ln,
+ msg,
+ warnings,
+ textual_msg=textual_msg,
+ can_attach_files=can_attach_files,
+ reply_to=comID,
+ relate_to_file=relate_file
+ )
+
else:
- try:
- raise InvenioWebCommentError(_('Comments on records have been disallowed by the administrator.'))
- except InvenioWebCommentError, exc:
- register_exception(req=req)
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
+ body = webcomment_templates.tmpl_error(
+ _('Comments on records have been disallowed by the '
+ 'administrator.'), ln)
+ return body
# check before submitting form
elif action == 'SUBMIT':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
if note.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
- try:
- raise InvenioWebCommentWarning(_('You must enter a title.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_ADD_NO_TITLE',))
+ warnings.append((_('You must enter a title.'), ''))
if score == 0 or score > 5:
- try:
- raise InvenioWebCommentWarning(_('You must choose a score.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(("WRN_WEBCOMMENT_ADD_NO_SCORE",))
+ warnings.append((_('You must choose a score.'), ''))
if msg.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
- try:
- raise InvenioWebCommentWarning(_('You must enter a text.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_ADD_NO_BODY',))
+ warnings.append((_('You must enter a text.'), ''))
# if no warnings, submit
if len(warnings) == 0:
if reviews:
if check_user_can_review(recID, client_ip_address, uid):
- success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
- note=note, score=score, priority=0,
- client_ip_address=client_ip_address,
- editor_type=editor_type,
- req=req,
- reply_to=comID)
+ success = query_add_comment_or_remark(
+ reviews,
+ recID=recID,
+ uid=uid,
+ msg=msg,
+ note=note,
+ score=score,
+ priority=0,
+ client_ip_address=client_ip_address,
+ editor_type=editor_type,
+ req=req,
+ reply_to=comID,
+ relate_file=relate_file
+ )
else:
- try:
- raise InvenioWebCommentWarning(_('You already wrote a review for this record.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append('WRN_WEBCOMMENT_CANNOT_REVIEW_TWICE')
+ warnings.append((_('You already wrote a review for '
+ 'this record.'), ''))
success = 1
else:
if check_user_can_comment(recID, client_ip_address, uid):
- success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
- note=note, score=score, priority=0,
- client_ip_address=client_ip_address,
- editor_type=editor_type,
- req=req,
-
- reply_to=comID, attached_files=attached_files)
+ success = query_add_comment_or_remark(
+ reviews,
+ recID=recID,
+ uid=uid,
+ msg=msg,
+ note=note,
+ score=score,
+ priority=0,
+ client_ip_address=client_ip_address,
+ editor_type=editor_type,
+ req=req,
+ reply_to=comID,
+ attached_files=attached_files,
+ relate_file=relate_file
+ )
if success > 0 and subscribe:
subscribe_user_to_discussion(recID, uid)
else:
- try:
- raise InvenioWebCommentWarning(_('You already posted a comment short ago. Please retry later.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append('WRN_WEBCOMMENT_TIMELIMIT')
+ warnings.append((_('You already posted a comment '
+ 'short ago. Please retry later.'), ''))
success = 1
if success > 0:
if CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL > 0:
notify_admin_of_new_comment(comID=success)
return webcomment_templates.tmpl_add_comment_successful(recID, ln, reviews, warnings, success)
else:
- try:
- raise InvenioWebCommentError(_('Failed to insert your comment to the database. Please try again.'))
- except InvenioWebCommentError, exc:
- register_exception(req=req)
- body = webcomment_templates.tmpl_error(exc.message, ln)
- return body
- #errors.append(('ERR_WEBCOMMENT_DB_INSERT_ERROR'))
+ register_exception(req=req)
+ body = webcomment_templates.tmpl_error(
+ _('Failed to insert your comment to the database.'
+ ' Please try again.'), ln)
+ return body
# if are warnings or if inserting comment failed, show user where warnings are
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
else:
- return webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files)
+ return webcomment_templates.tmpl_add_comment_form(
+ recID,
+ uid,
+ nickname,
+ ln,
+ msg,
+ warnings,
+ can_attach_files=can_attach_files,
+ relate_to_file=relate_file
+ )
# unknown action send to display
else:
- try:
- raise InvenioWebCommentWarning(_('Unknown action --> showing you the default add comment form.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning', req=req)
- warnings.append((exc.message, ''))
- #warnings.append(('WRN_WEBCOMMENT_ADD_UNKNOWN_ACTION',))
+ warnings.append((_('Unknown action --> showing you the default '
+ 'add comment form.'), ''))
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
else:
- return webcomment_templates.tmpl_add_comment_form(recID, uid, ln, msg, warnings, can_attach_files=can_attach_files)
+ return webcomment_templates.tmpl_add_comment_form(
+ recID,
+ uid,
+ nickname,
+ ln,
+ msg,
+ warnings,
+ can_attach_files=can_attach_files,
+ relate_to_file=relate_file
+ )
return ''
@@ -1756,30 +1877,25 @@ def notify_admin_of_new_comment(comID):
id_bibrec,
id_user,
body,
+ body_format,
date_creation,
- star_score, nb_votes_yes, nb_votes_total,
+ star_score, dummy, dummy,
title,
- nb_abuse_reports, round_name, restriction) = comment
+ dummy, dummy, dummy) = comment
else:
return
user_info = query_get_user_contact_info(id_user)
if len(user_info) > 0:
- (nickname, email, last_login) = user_info
+ (nickname, email, dummy) = user_info
if not len(nickname) > 0:
nickname = email.split('@')[0]
else:
- nickname = email = last_login = "ERROR: Could not retrieve"
+ nickname = email = "ERROR: Could not retrieve"
review_stuff = '''
Star score = %s
Title = %s''' % (star_score, title)
- washer = EmailWasher()
- try:
- body = washer.wash(body)
- except:
- body = cgi.escape(body)
-
record_info = webcomment_templates.tmpl_email_new_comment_admin(id_bibrec)
out = '''
The following %(comment_or_review)s has just been posted (%(date)s).
@@ -1796,11 +1912,14 @@ def notify_admin_of_new_comment(comID):
%(comment_or_review_caps)s:
%(comment_or_review)s ID = %(comID)s %(review_stuff)s
+ URL = <%(siteurl)s/%(CFG_SITE_RECORD)s/%(recID)s/%(comments_or_reviews)s/#C%(comID)s>
Body =
<--------------->
+
%(body)s
<--------------->
+
ADMIN OPTIONS:
To moderate the %(comment_or_review)s go to %(siteurl)s/%(CFG_SITE_RECORD)s/%(recID)s/%(comments_or_reviews)s/display?%(arguments)s
''' % \
@@ -1815,12 +1934,63 @@ def notify_admin_of_new_comment(comID):
'record_details' : record_info,
'comID' : comID2,
'review_stuff' : star_score > 0 and review_stuff or "",
- 'body' : body.replace('AUTHOR:
+
+ Nickname = %(nickname)s
+ Email = <%(email)s >
+ User ID = %(uid)s
+
+
+RECORD CONCERNED:
+
+
+%(comment_or_review_caps)s:
+
+
+<--------------->%(siteurl)s/%(CFG_SITE_RECORD)s/%(recID)s/%(comments_or_reviews)s/display?%(arguments)s >""" % {
+ 'comment_or_review' : star_score > 0 and 'review' or 'comment',
+ 'comment_or_review_caps': star_score > 0 and 'REVIEW' or 'COMMENT',
+ 'comments_or_reviews' : star_score > 0 and 'reviews' or 'comments',
+ 'date' : date_creation,
+ 'nickname' : nickname,
+ 'email' : email,
+ 'uid' : id_user,
+ 'recID' : id_bibrec,
+ 'record_details' : record_info,
+ 'comID' : comID2,
+ 'review_stuff' : star_score > 0 and review_stuff or "",
+ 'body' : webcomment_templates.tmpl_prepare_comment_body(body, body_format, CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["EMAIL"]),
+ 'siteurl' : CFG_SITE_SECURE_URL,
+ 'CFG_SITE_RECORD' : CFG_SITE_RECORD,
+ 'arguments' : 'ln=en&do=od#%s' % comID
+}
+ else:
+ html_content = None
+
from_addr = '%s WebComment <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(comID)
to_addrs = get_collection_moderators(comment_collection)
@@ -1831,7 +2001,13 @@ def notify_admin_of_new_comment(comID):
report_nums = ', '.join(report_nums)
subject = "A new comment/review has just been posted [%s|%s]" % (rec_collection, report_nums)
- send_email(from_addr, to_addrs, subject, out)
+ send_email(
+ fromaddr=from_addr,
+ toaddr=to_addrs,
+ subject=subject,
+ content=out,
+ html_content=html_content
+ )
def check_recID_is_in_range(recID, warnings=[], ln=CFG_SITE_LANG):
"""
@@ -1853,48 +2029,34 @@ def check_recID_is_in_range(recID, warnings=[], ln=CFG_SITE_LANG):
from invenio.search_engine import record_exists
success = record_exists(recID)
if success == 1:
- return (1,"")
+ return (1, "")
else:
- try:
- if success == -1:
- status = 'deleted'
- raise InvenioWebCommentWarning(_('The record has been deleted.'))
- else:
- status = 'inexistant'
- raise InvenioWebCommentWarning(_('Record ID %s does not exist in the database.') % recID)
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('ERR_WEBCOMMENT_RECID_INEXISTANT', recID))
- return (0, webcomment_templates.tmpl_record_not_found(status=status, recID=recID, ln=ln))
+ if success == -1:
+ status = 'deleted'
+ warning_message = _('The record has been deleted.')
+ else:
+ status = 'inexistant'
+ warning_message = _(
+ 'Record ID %s does not exist in the database.') % recID
+ warnings.append((warning_message, ''))
+ return (0, webcomment_templates.tmpl_record_not_found(
+ status=status, recID=recID, ln=ln))
elif recID == 0:
- try:
- raise InvenioWebCommentWarning(_('No record ID was given.'))
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('ERR_WEBCOMMENT_RECID_MISSING',))
- return (0, webcomment_templates.tmpl_record_not_found(status='missing', recID=recID, ln=ln))
+ warnings.append((_('No record ID was given.'), ''))
+ return (0, webcomment_templates.tmpl_record_not_found(
+ status='missing', recID=recID, ln=ln))
else:
- try:
- raise InvenioWebCommentWarning(_('Record ID %s is an invalid ID.') % recID)
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('ERR_WEBCOMMENT_RECID_INVALID', recID))
- return (0, webcomment_templates.tmpl_record_not_found(status='invalid', recID=recID, ln=ln))
+ warnings.append((_('Record ID %s is an invalid ID.') % recID, ''))
+ return (0, webcomment_templates.tmpl_record_not_found(
+ status='invalid', recID=recID, ln=ln))
else:
- try:
- raise InvenioWebCommentWarning(_('Record ID %s is not a number.') % recID)
- except InvenioWebCommentWarning, exc:
- register_exception(stream='warning')
- warnings.append((exc.message, ''))
- #warnings.append(('ERR_WEBCOMMENT_RECID_NAN', recID))
- return (0, webcomment_templates.tmpl_record_not_found(status='nan', recID=recID, ln=ln))
+ warnings.append((_('Record ID %s is not a number.') % recID, ''))
+ return (0, webcomment_templates.tmpl_record_not_found(
+ status='nan', recID=recID, ln=ln))
def check_int_arg_is_in_range(value, name, gte_value, lte_value=None):
"""
- Check that variable with name 'name' >= gte_value and optionally <= lte_value
+ Check that variable with name 'name' >= gte_value & optionally <= lte_value
@param value: variable value
@param name: variable name
@param errors: list of error tuples (error_id, value)
@@ -1904,34 +2066,17 @@ def check_int_arg_is_in_range(value, name, gte_value, lte_value=None):
"""
if type(value) is not int:
- try:
- raise InvenioWebCommentError('%s is not a number.' % value)
- except InvenioWebCommentError, exc:
- register_exception()
- body = webcomment_templates.tmpl_error(exc.message)
- return body
- #errors.append(('ERR_WEBCOMMENT_ARGUMENT_NAN', value))
- return 0
+ body = webcomment_templates.tmpl_error('%s is not a number.' % value)
+ return body
if value < gte_value:
- try:
- raise InvenioWebCommentError('%s invalid argument.' % value)
- except InvenioWebCommentError, exc:
- register_exception()
- body = webcomment_templates.tmpl_error(exc.message)
- return body
- #errors.append(('ERR_WEBCOMMENT_ARGUMENT_INVALID', value))
- return 0
+ body = webcomment_templates.tmpl_error('%s invalid argument.' % value)
+ return body
if lte_value:
if value > lte_value:
- try:
- raise InvenioWebCommentError('%s invalid argument.' % value)
- except InvenioWebCommentError, exc:
- register_exception()
- body = webcomment_templates.tmpl_error(exc.message)
- return body
- #errors.append(('ERR_WEBCOMMENT_ARGUMENT_INVALID', value))
- return 0
+ body = webcomment_templates.tmpl_error(
+ '%s invalid argument.' % value)
+ return body
return 1
def get_mini_reviews(recid, ln=CFG_SITE_LANG):
@@ -1987,7 +2132,7 @@ def check_user_can_view_comment(user_info, comid, restriction=None):
if restriction is None:
comment = query_get_comment(comid)
if comment:
- restriction = comment[11]
+ restriction = comment[12]
else:
return (1, 'Comment %i does not exist' % comid)
if restriction == "":
@@ -2033,7 +2178,7 @@ def check_user_can_attach_file_to_comments(user_info, recid):
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'attachcommentfile', authorized_if_no_roles=False, collection=record_primary_collection)
-def toggle_comment_visibility(uid, comid, collapse, recid):
+def toggle_comment_visibility(uid, comid, collapse, recid, force=False):
"""
Toggle the visibility of the given comment (collapse) for the
given user. Return the new visibility
@@ -2042,6 +2187,7 @@ def toggle_comment_visibility(uid, comid, collapse, recid):
@param comid: the comment id to close/open
@param collapse: if the comment is to be closed (1) or opened (0)
@param recid: the record id to which the comment belongs
+ @param force: if we need to delete previous comment state
@return: if the comment is visible or not after the update
"""
# We rely on the client to tell if comment should be collapsed or
@@ -2057,24 +2203,33 @@ def toggle_comment_visibility(uid, comid, collapse, recid):
# when deleting an entry, as in the worst case no line would be
# removed. For optimized retrieval of row to delete, the id_bibrec
# column is used, though not strictly necessary.
- if collapse:
- query = """SELECT id_bibrec from cmtRECORDCOMMENT WHERE id=%s"""
- params = (comid,)
- res = run_sql(query, params)
- if res:
- query = """INSERT IGNORE INTO cmtCOLLAPSED (id_bibrec, id_cmtRECORDCOMMENT, id_user)
- VALUES (%s, %s, %s)"""
- params = (res[0][0], comid, uid)
+
+ # Split all comment ids
+ splited_comment_ids = comid.split(',')
+ # `False` if the comment is hidden else `True` if the comment is visible
+ comment_state = False
+ if collapse and force or not collapse:
+ for comment_id in splited_comment_ids:
+ query = """DELETE FROM cmtCOLLAPSED WHERE
+ id_cmtRECORDCOMMENT=%s and
+ id_user=%s and
+ id_bibrec=%s"""
+ params = (comment_id, uid, recid)
run_sql(query, params)
- return True
- else:
- query = """DELETE FROM cmtCOLLAPSED WHERE
- id_cmtRECORDCOMMENT=%s and
- id_user=%s and
- id_bibrec=%s"""
- params = (comid, uid, recid)
- run_sql(query, params)
- return False
+
+ if collapse:
+ for comment_id in splited_comment_ids:
+ query = """SELECT id_bibrec from cmtRECORDCOMMENT WHERE id=%s"""
+ params = (comment_id,)
+ res = run_sql(query, params)
+ if res:
+ query = """INSERT IGNORE INTO cmtCOLLAPSED (id_bibrec, id_cmtRECORDCOMMENT, id_user)
+ VALUES (%s, %s, %s)"""
+ params = (res[0][0], comment_id, uid)
+ run_sql(query, params)
+ comment_state = True
+ return comment_state
+
def get_user_collapsed_comments_for_record(uid, recid):
"""
@@ -2160,9 +2315,9 @@ def perform_display_your_comments(user_info,
elif selected_order_by_option == "ocf":
query_params += " ORDER BY date_creation ASC"
elif selected_order_by_option == "grlf":
- query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT max(date_creation) as maxdatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.maxdatecreation DESC, cmt.date_creation DESC"
+ query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.body_format, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT max(date_creation) as maxdatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.maxdatecreation DESC, cmt.date_creation DESC"
elif selected_order_by_option == "grof":
- query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT min(date_creation) as mindatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.mindatecreation ASC"
+ query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.body_format, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT min(date_creation) as mindatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.mindatecreation ASC"
if selected_display_number_option.isdigit():
selected_display_number_option_as_int = int(selected_display_number_option)
@@ -2180,7 +2335,7 @@ def perform_display_your_comments(user_info,
if selected_order_by_option in ("grlf", "grof"):
res = run_sql(query + query_params, (user_info['uid'], user_info['uid']))
else:
- res = run_sql("SELECT id_bibrec, id, date_creation, body, status, in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0" + query_params, (user_info['uid'], ))
+ res = run_sql("SELECT id_bibrec, id, date_creation, body, body_format, status, in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0" + query_params, (user_info['uid'], ))
return webcomment_templates.tmpl_your_comments(user_info, res,
page_number=page_number,
@@ -2190,3 +2345,20 @@ def perform_display_your_comments(user_info,
nb_total_results=nb_total_results,
nb_total_pages=nb_total_pages,
ln=ln)
+
+def account_user_comments(uid, ln = CFG_SITE_LANG):
+ """
+ Information on the user comments for the "Your Account" page.
+ """
+
+ query = """ SELECT COUNT(id)
+ FROM cmtRECORDCOMMENT
+ WHERE id_user = %s
+ AND star_score = 0"""
+ params = (uid,)
+ result = run_sql(query, params)
+ comments = result[0][0]
+
+ out = webcomment_templates.tmpl_account_user_comments(comments, ln)
+
+ return out
diff --git a/modules/webcomment/lib/webcomment_config.py b/modules/webcomment/lib/webcomment_config.py
index 485b9f7dbd..52549fd8ab 100644
--- a/modules/webcomment/lib/webcomment_config.py
+++ b/modules/webcomment/lib/webcomment_config.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-#
+
# This file is part of Invenio.
# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
#
@@ -23,6 +23,10 @@
__revision__ = "$Id$"
+from invenio.config import CFG_CERN_SITE
+from invenio.search_engine_utils import get_fieldvalues
+from invenio.webuser import collect_user_info
+
CFG_WEBCOMMENT_ACTION_CODE = {
'ADD_COMMENT': 'C',
'ADD_REVIEW': 'R',
@@ -30,22 +34,122 @@
'REPORT_ABUSE': 'A'
}
+CFG_WEBCOMMENT_BODY_FORMATS = {
+ "HTML": "HTML",
+ "TEXT": "TXT",
+ "MARKDOWN": "MD",
+}
+
+CFG_WEBCOMMENT_OUTPUT_FORMATS = {
+ "HTML": {
+ "WEB": "WEB",
+ "EMAIL": "HTML_EMAIL",
+ "CKEDITOR": "CKEDITOR",
+ },
+ "TEXT": {
+ "EMAIL": "TEXT_EMAIL",
+ "TEXTAREA": "TEXTAREA",
+ },
+}
+
+# Based on CFG_WEBCOMMENT_DEADLINE_CONFIGURATION we can display, but not
+# enforce comment submission deadlines. The configuration is composed of rules
+# (dictionary items). For a rule to be applied in the currently displayed
+# record, the dictionary key has to be in the list of values of the MARC field
+# that is the first element of the tuple in that dictionary key's value. If
+# that is the case, then the dealine is retrieved as the first value of the
+# MARC field that is the second element of the tuple in that dictionary key's
+# value. In order to programmatically check if the deadline has passed or not
+# we need to know the format of the given deadline, using standard strftime
+# conversion specifications . The deadline
+# format is the third element of the tuple in that dictionary key's value.
+if CFG_CERN_SITE:
+ CFG_WEBCOMMENT_DEADLINE_CONFIGURATION = {
+ "ATLASPUBDRAFT": (
+ "980__a",
+ "925__b",
+ "%d %b %Y",
+ )
+ }
+else:
+ CFG_WEBCOMMENT_DEADLINE_CONFIGURATION = {
+ }
+
+
+def check_user_is_editor(uid, record_id, data):
+ """Check if the user is editor.
+
+ :param int uid: The user id
+ :param int record_id: The record id
+ :param dict data: Extra arguments
+ :return: If the user is editor
+ :rtype: bool
+
+ .. note::
+
+ The report number is been splited and wrapped with proper suffix and
+ prefix for matching CERN's e-groups.
+
+ """
+ report_number_field = data.get('report_number_field')
+ report_number = get_fieldvalues(record_id, report_number_field)
+
+ if report_number:
+ report_number = '-'.join(report_number[0].split('-')[1:-1])
+ the_list = "{0}-{1}-{2}".format(
+ data.get('prefix'), report_number.lower(), data.get('suffix')
+ )
+ user_info = collect_user_info(uid)
+ user_lists = user_info.get('group', [])
+ if the_list in user_lists:
+ return True
+ return False
+
+# Based on CFG_WEBCOMMENT_USER_EDITOR we can display an extra html element
+# for users which are editors. The configuration uses the the collection name
+# as a key which holds a tuple with two items. The first one is the MARC field
+# which holds the collection and the seccond one is a dictionary. The
+# dictionary *MUST* contain a key called `callback` which holds the check
+# function. The check function *MUST* have `user_id` as first argument, the
+# `record_id` as second and a third which contains any other data.
+# Read more `~webcomment_config.check_user_is_editor`
+if CFG_CERN_SITE:
+ CFG_WEBCOMMENT_EXTRA_CHECKBOX = {
+ "ATLAS": (
+ "980__a",
+ dict(
+ report_number_field="037__a",
+ label="Post comment as Editor's response",
+ callback=check_user_is_editor,
+ prefix="atlas",
+ suffix="editor [cern]",
+ value="Editor response",
+ )
+ )
+ }
+else:
+ CFG_WEBCOMMENT_EXTRA_CHECKBOX = {}
+
+
# Exceptions: errors
class InvenioWebCommentError(Exception):
"""A generic error for WebComment."""
def __init__(self, message):
"""Initialisation."""
self.message = message
+
def __str__(self):
"""String representation."""
return repr(self.message)
+
# Exceptions: warnings
class InvenioWebCommentWarning(Exception):
"""A generic warning for WebComment."""
def __init__(self, message):
"""Initialisation."""
self.message = message
+
def __str__(self):
"""String representation."""
return repr(self.message)
diff --git a/modules/webcomment/lib/webcomment_dblayer.py b/modules/webcomment/lib/webcomment_dblayer.py
new file mode 100644
index 0000000000..83c889138c
--- /dev/null
+++ b/modules/webcomment/lib/webcomment_dblayer.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+# This file is part of Invenio.
+# Copyright (C) 2014 CERN.
+#
+# Invenio is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# Invenio is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Invenio; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+""" WebComment database layer """
+
+__revision__ = "$Id$"
+
+from invenio.dbquery import run_sql
+from invenio.bibdocfile import BibRecDocs
+
+
+def get_comment_to_bibdoc_relations(recID):
+ """
+ Retrieve all comment to to bibdoc relations for the given record.
+
+ and bibdocfiles they refer to.
+ :param recID: Id of the record
+ :return: correlations between comments and bibdocfiles
+ """
+ query = """
+ SELECT id_bibrec,
+ id_cmtRECORDCOMMENT,
+ id_bibdoc,
+ version
+ FROM cmtRECORDCOMMENT_bibdoc
+ WHERE id_bibrec = %s
+ """
+
+ comments_to_bibdoc = run_sql(query, (recID,), with_dict=True)
+ brd = BibRecDocs(recID)
+ bds = brd.list_bibdocs()
+ res = []
+ for bd in bds:
+ for comments in comments_to_bibdoc:
+ if comments['id_bibdoc'] == bd.id:
+ res.append({
+ 'id_comment': comments['id_cmtRECORDCOMMENT'],
+ 'id_bibdoc': bd.id,
+ 'version': comments['version'],
+ 'docname': brd.get_docname(bd.id),
+ })
+ return res
+
+
+def set_comment_to_bibdoc_relation(redID, cmtID, bibdocfileID, version):
+ """
+ Set a comment to bibdoc relation.
+
+ :param redID: Id of the record
+ :param cmtID: Id of the comment
+ :param bibdocfileID: Id of the bibdocfile
+ """
+
+ query = """
+ INSERT INTO cmtRECORDCOMMENT_bibdoc
+ (id_bibrec,
+ id_cmtRECORDCOMMENT,
+ id_bibdoc,
+ version)
+ VALUES (%s,
+ %s,
+ %s,
+ %s)
+ """
+
+ parameters = (redID, cmtID, bibdocfileID, version)
+
+ result = run_sql(query, parameters)
+
+ return result
diff --git a/modules/webcomment/lib/webcomment_templates.py b/modules/webcomment/lib/webcomment_templates.py
index d47089e80c..413ddad4e8 100644
--- a/modules/webcomment/lib/webcomment_templates.py
+++ b/modules/webcomment/lib/webcomment_templates.py
@@ -1,8 +1,7 @@
# -*- coding: utf-8 -*-
-# Comments and reviews for records.
# This file is part of Invenio.
-# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 CERN.
+# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
@@ -23,34 +22,57 @@
__revision__ = "$Id$"
import cgi
-
-# Invenio imports
+import re
+import html2text
+import markdown2
+import json
+from datetime import datetime, date
+
+from invenio.webcomment_config import (
+ CFG_WEBCOMMENT_BODY_FORMATS,
+ CFG_WEBCOMMENT_OUTPUT_FORMATS,
+ CFG_WEBCOMMENT_EXTRA_CHECKBOX
+)
from invenio.urlutils import create_html_link, create_url
-from invenio.webuser import get_user_info, collect_user_info, isGuestUser, get_email
-from invenio.dateutils import convert_datetext_to_dategui
+from invenio.webuser import (
+ get_user_info,
+ collect_user_info,
+ isGuestUser,
+ get_email
+)
+from invenio.dateutils import (
+ convert_datetext_to_dategui,
+ convert_datestruct_to_dategui
+)
from invenio.webmessage_mailutils import email_quoted_txt2html
-from invenio.config import CFG_SITE_URL, \
- CFG_SITE_SECURE_URL, \
- CFG_BASE_URL, \
- CFG_SITE_LANG, \
- CFG_SITE_NAME, \
- CFG_SITE_NAME_INTL,\
- CFG_SITE_SUPPORT_EMAIL,\
- CFG_WEBCOMMENT_ALLOW_REVIEWS, \
- CFG_WEBCOMMENT_ALLOW_COMMENTS, \
- CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR, \
- CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN, \
- CFG_WEBCOMMENT_AUTHOR_DELETE_COMMENT_OPTION, \
- CFG_CERN_SITE, \
- CFG_SITE_RECORD, \
- CFG_WEBCOMMENT_MAX_ATTACHED_FILES, \
- CFG_WEBCOMMENT_MAX_ATTACHMENT_SIZE
+from invenio.config import \
+ CFG_SITE_URL, \
+ CFG_SITE_SECURE_URL, \
+ CFG_BASE_URL, \
+ CFG_SITE_LANG, \
+ CFG_SITE_NAME, \
+ CFG_SITE_NAME_INTL,\
+ CFG_SITE_SUPPORT_EMAIL,\
+ CFG_WEBCOMMENT_ALLOW_REVIEWS, \
+ CFG_WEBCOMMENT_ALLOW_COMMENTS, \
+ CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR, \
+ CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN, \
+ CFG_WEBCOMMENT_AUTHOR_DELETE_COMMENT_OPTION, \
+ CFG_CERN_SITE, \
+ CFG_SITE_RECORD, \
+ CFG_WEBCOMMENT_MAX_ATTACHED_FILES, \
+ CFG_WEBCOMMENT_MAX_ATTACHMENT_SIZE, \
+ CFG_WEBCOMMENT_ENABLE_HTML_EMAILS, \
+ CFG_WEBCOMMENT_ENABLE_MARKDOWN_TEXT_RENDERING
from invenio.htmlutils import get_html_text_editor, create_html_select
from invenio.messages import gettext_set_language
from invenio.bibformat import format_record
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_admin import acc_get_user_roles_from_user_info, acc_get_role_id
+from invenio.bibdocfile import BibRecDocs
from invenio.search_engine_utils import get_fieldvalues
+from invenio.webcomment_config import CFG_WEBCOMMENT_DEADLINE_CONFIGURATION
+
class Template:
"""templating class, refer to webcomment.py for examples of call"""
@@ -75,7 +97,8 @@ def tmpl_get_first_comments_without_ranking(self, recID, ln, comments, nb_commen
c_body = 3
c_status = 4
c_nb_reports = 5
- c_id = 6
+ c_id = 7
+ c_body_format = 11
warnings = self.tmpl_warnings(warnings, ln)
@@ -119,6 +142,7 @@ def tmpl_get_first_comments_without_ranking(self, recID, ln, comments, nb_commen
comment_uid=comment[c_user_id],
date_creation=comment[c_date_creation],
body=comment[c_body],
+ body_format=comment[c_body_format],
status=comment[c_status],
nb_reports=comment[c_nb_reports],
reply_link=reply_link,
@@ -232,6 +256,7 @@ def tmpl_get_first_comments_with_ranking(self, recID, ln, comments=None, nb_comm
c_star_score = 8
c_title = 9
c_id = 10
+ c_body_format = 14
warnings = self.tmpl_warnings(warnings, ln)
@@ -282,6 +307,7 @@ def tmpl_get_first_comments_with_ranking(self, recID, ln, comments=None, nb_comm
comment_uid=comment[c_user_id],
date_creation=comment[c_date_creation],
body=comment[c_body],
+ body_format=comment[c_body_format],
status=comment[c_status],
nb_reports=comment[c_nb_reports],
nb_votes_total=comment[c_nb_votes_total],
@@ -362,7 +388,30 @@ def tmpl_get_first_comments_with_ranking(self, recID, ln, comments=None, nb_comm
write_button_form)
return out
- def tmpl_get_comment_without_ranking(self, req, ln, nickname, comment_uid, date_creation, body, status, nb_reports, reply_link=None, report_link=None, undelete_link=None, delete_links=None, unreport_link=None, recID=-1, com_id='', attached_files=None, collapsed_p=False, admin_p=False):
+ def tmpl_get_comment_without_ranking(
+ self,
+ req,
+ ln,
+ nickname,
+ comment_uid,
+ date_creation,
+ body,
+ body_format,
+ status,
+ nb_reports,
+ cmt_title,
+ reply_link=None,
+ report_link=None,
+ undelete_link=None,
+ delete_links=None,
+ unreport_link=None,
+ recID=-1,
+ com_id='',
+ attached_files=None,
+ collapsed_p=False,
+ admin_p=False,
+ related_files=None
+ ):
"""
private function
@param req: request object to fetch user info
@@ -384,6 +433,7 @@ def tmpl_get_comment_without_ranking(self, req, ln, nickname, comment_uid, date_
@param com_id: ID of the comment displayed
@param attached_files: list of attached files
@param collapsed_p: if the comment should be collapsed or not
+ @param related_files: Display related files
@return: html table of comment
"""
from invenio.search_engine import guess_primary_collection_of_a_record
@@ -394,7 +444,13 @@ def tmpl_get_comment_without_ranking(self, req, ln, nickname, comment_uid, date_
if attached_files is None:
attached_files = []
out = ''
- final_body = email_quoted_txt2html(body)
+
+ final_body = self.tmpl_prepare_comment_body(
+ body,
+ body_format,
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["WEB"]
+ )
+
title = nickname
title += '
+ This comment is related with file %(docname)s, version %(version)s
+
+ """ % {
+ 'docname': related_file['docname'],
+ 'version': related_file['version'],
+ 'id_bibdoc': related_file['id_bibdoc']
+ }
+ except (TypeError, KeyError):
+ pass
+
+ title_element = ""
+ if cmt_title:
+ title_element = "".format(
+ cmt_title)
+
out += """
""" % {
+ 'title': title,
+ 'body': final_body,
+ 'links': links,
+ 'attached_files_html': attached_files_html,
+ 'date': date_creation,
+ 'site_url': CFG_SITE_URL,
+ 'comid': com_id,
+ 'collapsible_content_style': collapsed_p and 'display:none' or '',
+ 'toggle_visibility_block': toggle_visibility_block,
+ 'related_file_element': related_file_element,
+ 'title_element': title_element,
+ }
return out
- def tmpl_get_comment_with_ranking(self, req, ln, nickname, comment_uid, date_creation, body, status, nb_reports, nb_votes_total, nb_votes_yes, star_score, title, report_link=None, delete_links=None, undelete_link=None, unreport_link=None, recID=-1, admin_p=False):
+ def tmpl_get_comment_with_ranking(
+ self,
+ req,
+ ln,
+ nickname,
+ comment_uid,
+ date_creation,
+ body,
+ body_format,
+ status,
+ nb_reports,
+ nb_votes_total,
+ nb_votes_yes,
+ star_score,
+ title,
+ report_link=None,
+ delete_links=None,
+ undelete_link=None,
+ unreport_link=None,
+ recID=-1,
+ admin_p=False,
+ related_files=None
+ ):
"""
private function
@param req: request object to fetch user info
@@ -518,6 +619,7 @@ def tmpl_get_comment_with_ranking(self, req, ln, nickname, comment_uid, date_cre
@param delete_link: http link to delete the message
@param unreport_link: http link to unreport the comment
@param recID: recID where the comment is posted
+ @param related_files: Related comment files
@return: html table of review
"""
from invenio.search_engine import guess_primary_collection_of_a_record
@@ -531,6 +633,12 @@ def tmpl_get_comment_with_ranking(self, req, ln, nickname, comment_uid, date_cre
out = ""
+ final_body = self.tmpl_prepare_comment_body(
+ body,
+ body_format,
+ CFG_WEBCOMMENT_OUTPUT_FORMATS["HTML"]["WEB"]
+ )
+
date_creation = convert_datetext_to_dategui(date_creation, ln=ln)
reviewed_label = _("Reviewed by %(x_nickname)s on %(x_date)s") % {'x_nickname': nickname, 'x_date':date_creation}
## FIX
@@ -541,10 +649,10 @@ def tmpl_get_comment_with_ranking(self, req, ln, nickname, comment_uid, date_cre
links = ''
_body = ''
if body != '':
- _body = '''
-
-%s
- ''' % email_quoted_txt2html(body, linebreak_html='')
+ _body = '''
+
+ %s
+
''' % final_body
# Check if user is a comment moderator
record_primary_collection = guess_primary_collection_of_a_record(recID)
@@ -577,10 +685,12 @@ def tmpl_get_comment_with_ranking(self, req, ln, nickname, comment_uid, date_cre
else:
_body = ''
links = ''
+ related_file_element = ''
out += '''