From 3746d6fe8de8e1db654f9ddf1744d484ee482155 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 17 Mar 2026 14:56:48 +0100 Subject: [PATCH 01/10] Use `Compromiser` evaluator. See #MOD-1065 --- CHANGES.rst | 4 ++-- src/collective/documentgenerator/browser/generation_view.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 31d88356..0c5e5565 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,8 +4,8 @@ Changelog 3.48 (unreleased) ----------------- -- Nothing changed yet. - +- Use `Compromiser` evaluator. + [gbastien] 3.47 (2026-03-03) ----------------- diff --git a/src/collective/documentgenerator/browser/generation_view.py b/src/collective/documentgenerator/browser/generation_view.py index 6a9aa230..f5dcac6b 100644 --- a/src/collective/documentgenerator/browser/generation_view.py +++ b/src/collective/documentgenerator/browser/generation_view.py @@ -4,6 +4,7 @@ from AccessControl import Unauthorized from appy.pod.renderer import CsvOptions from appy.pod.renderer import Renderer +from appy.pod.evaluator import Compromiser from appy.pod.styles_manager import TableProperties from collective.documentgenerator import config from collective.documentgenerator import utils @@ -283,6 +284,7 @@ def _render_document(self, pod_template, output_format, sub_documents, raiseOnEr html=True, optimalColumnWidths=optimalColumnWidths, distributeColumns=distributeColumns, + evaluator=Compromiser(), stylesMapping=stylesMapping, stream=config.get_use_stream(), csvOptions=csvOptions, From 4bef38d8b8a657d012ec2ad6b9f8d06ba020d8a2 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 13:00:17 +0100 Subject: [PATCH 02/10] Added migration to `16` that will make sure existing POD templates are safe. Added parameter `prepend_pod_title=True` to `DocumentGenerationView._get_filename` so it is especially possible to not preprend pod template title to generated file filename. --- CHANGES.rst | 6 ++ .../browser/generation_view.py | 16 +++- .../migrations/migrate_to_16.py | 90 +++++++++++++++++++ .../documentgenerator/profiles.zcml | 11 +++ .../profiles/plone4/metadata.xml | 2 +- .../profiles/plone5/metadata.xml | 2 +- 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 src/collective/documentgenerator/migrations/migrate_to_16.py diff --git a/CHANGES.rst b/CHANGES.rst index 0c5e5565..1834b00b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,12 @@ Changelog - Use `Compromiser` evaluator. [gbastien] +- Added migration to `16` that will make sure existing POD templates are safe. + [gbastien] +- Added parameter `prepend_pod_title=True` to `DocumentGenerationView._get_filename` + so it is especially possible to not preprend pod template title to generated + file filename. + [gbastien] 3.47 (2026-03-03) ----------------- diff --git a/src/collective/documentgenerator/browser/generation_view.py b/src/collective/documentgenerator/browser/generation_view.py index f5dcac6b..ab912932 100644 --- a/src/collective/documentgenerator/browser/generation_view.py +++ b/src/collective/documentgenerator/browser/generation_view.py @@ -128,13 +128,21 @@ def _generate_doc(self, pod_template, output_format, **kwargs): filename = self._get_filename() return rendered, filename, gen_context - def _get_filename(self): - """ """ - # we limit filename to 120 characters - first_part = u'{0} {1}'.format(self.pod_template.title, safe_unicode(self.context.Title())) + def _get_filename(self, prepend_pod_title=True): + """ + Return generated file filename. + + :param prepend_pod_title: will preprend the POD template title before + context title for the generated file filename + title then pod template title + """ + first_part = safe_unicode(self.context.Title()) + if prepend_pod_title: + first_part = u'{0} {1}'.format(self.pod_template.title, first_part) # replace unicode special characters with ascii equivalent value first_part = unicodedata.normalize('NFKD', first_part).encode('ascii', 'ignore') util = queryUtility(IFileNameNormalizer) + # we limit filename to 120 characters # remove '-' from first_part because it is handled by cropName that manages max_length # and it behaves weirdly if it encounters '-' # moreover avoid more than one blank space at a time diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py new file mode 100644 index 00000000..3192f233 --- /dev/null +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +from collections import OrderedDict +from collective.documentgenerator.content.pod_template import IPODTemplate +from collective.documentgenerator.search_replace.pod_template import SearchAndReplacePODTemplates +from imio.helpers.content import uuidToObject +from imio.migrator.migrator import Migrator +from imio.pyutils.utils import safe_encode +from plone import api + +import logging + + +logger = logging.getLogger('collective.documentgenerator') + + +class Migrate_To_14(Migrator): + + def __init__(self, context): + Migrator.__init__(self, context) + self.catalog = api.portal.get_tool('portal_catalog') + + def _clean_expr(self): + """Clean expressions of existing POD templates.""" + logger.info('Cleaning expressions for existing POD templates...') + results = [] + for brain in self.catalog(object_provides=IPODTemplate.__identifier__): + pod_template = brain.getObject() + with SearchAndReplacePODTemplates([pod_template]) as search_replace: + res = search_replace.replace('_banned_', '++REPLACED++', is_regex=False) + if res: + results.append(res) + res = search_replace.replace('_underscored_', '++REPLACED++', is_regex=False) + if res: + results.append(res) + # format results and dump it in the Zope log + # as clean as possible so it can be used to know what changed + data = {} + for result in results: + pt_uid, infos = result.items()[0] + pt = uuidToObject(pt_uid, unrestricted=True) + pt_path_and_title = "{0} - {1}".format( + '/'.join(pt.getPhysicalPath()), pt.Title()) + if pt_path_and_title not in data: + data[pt_path_and_title] = [] + self.warnings.append('Replacements were done in POD template at %s' + % pt_path_and_title) + for info in infos: + # collective.documentgenerator < 3.30 from which we use appy.pod S&R + # XXX to be removed when using collective.documentgenerator >= 3.30 + if hasattr(info, 'pod_expr'): + data[pt_path_and_title].append("---- " + info.pod_expr) + data[pt_path_and_title].append("++++ " + info.new_pod_expr) + else: + line = repr(info).replace(' These changes were done:', '>>>'). \ + replace('\n\n', '\n').rstrip('\n') + data[pt_path_and_title].append(line) + logger.info("REPLACEMENTS IN POD TEMPLATES") + if not data: + logger.info("=============================") + logger.info("No replacement was done.") + else: + # order data by pt_path + ordered_data = OrderedDict(sorted(data.items())) + output = ["============================="] + for pt_path_and_title, infos in ordered_data.items(): + output.append('\n') + output.append("POD template " + pt_path_and_title) + output.append('-' * len("POD template " + pt_path_and_title)) + for info in infos: + output.append(info) + output.append('\n') + # make sure we do not mix unicode and utf-8 + fixed_output = [] + for line in output: + line = safe_encode(line) + fixed_output.append(line) + logger.info('\n'.join(fixed_output)) + logger.info('Done.') + + def run(self): + logger.info('Migrating to collective.documentgenerator 14...') + self._clean_expr() + self.finish() + + +def migrate(context): + ''' + ''' + Migrate_To_14(context).run() diff --git a/src/collective/documentgenerator/profiles.zcml b/src/collective/documentgenerator/profiles.zcml index 22739827..8c55aa75 100644 --- a/src/collective/documentgenerator/profiles.zcml +++ b/src/collective/documentgenerator/profiles.zcml @@ -180,6 +180,17 @@ import_steps="plone.app.registry" /> + + + + + diff --git a/src/collective/documentgenerator/profiles/plone4/metadata.xml b/src/collective/documentgenerator/profiles/plone4/metadata.xml index fa919fd1..5086056f 100644 --- a/src/collective/documentgenerator/profiles/plone4/metadata.xml +++ b/src/collective/documentgenerator/profiles/plone4/metadata.xml @@ -3,5 +3,5 @@ profile-collective.documentgenerator:install-base - 15 + 16 diff --git a/src/collective/documentgenerator/profiles/plone5/metadata.xml b/src/collective/documentgenerator/profiles/plone5/metadata.xml index fa919fd1..5086056f 100644 --- a/src/collective/documentgenerator/profiles/plone5/metadata.xml +++ b/src/collective/documentgenerator/profiles/plone5/metadata.xml @@ -3,5 +3,5 @@ profile-collective.documentgenerator:install-base - 15 + 16 From b547acbeddfe02cc54a37e8e0c69f5863be12820 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 13:44:39 +0100 Subject: [PATCH 03/10] Try with LO7.3 --- .github/workflows/python-package.yml | 2 +- .../documentgenerator/migrations/migrate_to_16.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c90af0c3..cf9ad538 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: PLONE_VERSION: [ 4.3, 5.1, 5.2 ] services: libreoffice: - image: harbor.imio.be/library/libreoffice:25.2 + image: harbor.imio.be/library/libreoffice:7.3 ports: - 2002:2002 volumes: diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py index 3192f233..12464ae8 100644 --- a/src/collective/documentgenerator/migrations/migrate_to_16.py +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -14,7 +14,7 @@ logger = logging.getLogger('collective.documentgenerator') -class Migrate_To_14(Migrator): +class Migrate_To_16(Migrator): def __init__(self, context): Migrator.__init__(self, context) @@ -27,12 +27,14 @@ def _clean_expr(self): for brain in self.catalog(object_provides=IPODTemplate.__identifier__): pod_template = brain.getObject() with SearchAndReplacePODTemplates([pod_template]) as search_replace: - res = search_replace.replace('_banned_', '++REPLACED++', is_regex=False) + res = search_replace.replace('_underscored_', '++REPLACED++', is_regex=False) if res: results.append(res) - res = search_replace.replace('_underscored_', '++REPLACED++', is_regex=False) + with SearchAndReplacePODTemplates([pod_template]) as search_replace: + res = search_replace.replace('_banned_', '++REPLACED++', is_regex=False) if res: results.append(res) + import ipdb; ipdb.set_trace() # format results and dump it in the Zope log # as clean as possible so it can be used to know what changed data = {} @@ -79,7 +81,7 @@ def _clean_expr(self): logger.info('Done.') def run(self): - logger.info('Migrating to collective.documentgenerator 14...') + logger.info('Migrating to collective.documentgenerator 16...') self._clean_expr() self.finish() @@ -87,4 +89,4 @@ def run(self): def migrate(context): ''' ''' - Migrate_To_14(context).run() + Migrate_To_16(context).run() From 2d82d0ce3f31d7396ab3cc6c37380104baf44c23 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 14:06:01 +0100 Subject: [PATCH 04/10] Grumpf --- src/collective/documentgenerator/migrations/migrate_to_16.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py index 12464ae8..11c94227 100644 --- a/src/collective/documentgenerator/migrations/migrate_to_16.py +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -34,7 +34,6 @@ def _clean_expr(self): res = search_replace.replace('_banned_', '++REPLACED++', is_regex=False) if res: results.append(res) - import ipdb; ipdb.set_trace() # format results and dump it in the Zope log # as clean as possible so it can be used to know what changed data = {} From 360bfddc01a6ad831ef539161f4cbb33ad0d123f Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 14:25:56 +0100 Subject: [PATCH 05/10] Print soffice version --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cf9ad538..286614f6 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -35,6 +35,7 @@ jobs: sudo apt-get install -qq -y libreoffice libreoffice-script-provider-python libjpeg62 libjpeg62-dev libbz2-dev mkdir -p buildout-cache/{eggs,downloads} pip install -r requirements.txt + soffice --version - name: Set up pyenv and Python uses: "gabrielfalcao/pyenv-action@v18" with: From bf457b5c6dbc22ecd67e16579febc3260068a064 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 14:46:24 +0100 Subject: [PATCH 06/10] Use LO 25.8 --- .github/workflows/python-package.yml | 4 ++-- .../migrations/migrate_to_16.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 286614f6..c3d57efb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: PLONE_VERSION: [ 4.3, 5.1, 5.2 ] services: libreoffice: - image: harbor.imio.be/library/libreoffice:7.3 + image: harbor.imio.be/library/libreoffice:25.8 ports: - 2002:2002 volumes: @@ -72,7 +72,7 @@ jobs: PLONE_VERSION: [5.2] services: libreoffice: - image: harbor.imio.be/library/libreoffice:25.2 + image: harbor.imio.be/library/libreoffice:25.8 ports: - 2002:2002 volumes: diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py index 11c94227..fd4f1a42 100644 --- a/src/collective/documentgenerator/migrations/migrate_to_16.py +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from appy.pod.evaluator import Compromiser from collections import OrderedDict from collective.documentgenerator.content.pod_template import IPODTemplate from collective.documentgenerator.search_replace.pod_template import SearchAndReplacePODTemplates @@ -27,13 +28,14 @@ def _clean_expr(self): for brain in self.catalog(object_provides=IPODTemplate.__identifier__): pod_template = brain.getObject() with SearchAndReplacePODTemplates([pod_template]) as search_replace: - res = search_replace.replace('_underscored_', '++REPLACED++', is_regex=False) - if res: - results.append(res) - with SearchAndReplacePODTemplates([pod_template]) as search_replace: - res = search_replace.replace('_banned_', '++REPLACED++', is_regex=False) - if res: - results.append(res) + for banned_expr in Compromiser.banned: + res = search_replace.replace(banned_expr, '++REPLACED++', is_regex=False) + if res: + results.append(res) + for underscored_regex in Compromiser.underscored: + res = search_replace.replace(underscored_regex.pattern, '++REPLACED++', is_regex=True) + if res: + results.append(res) # format results and dump it in the Zope log # as clean as possible so it can be used to know what changed data = {} @@ -49,6 +51,7 @@ def _clean_expr(self): for info in infos: # collective.documentgenerator < 3.30 from which we use appy.pod S&R # XXX to be removed when using collective.documentgenerator >= 3.30 + import ipdb; ipdb.set_trace() if hasattr(info, 'pod_expr'): data[pt_path_and_title].append("---- " + info.pod_expr) data[pt_path_and_title].append("++++ " + info.new_pod_expr) From 807c112a50f32a248e0023fff0c0a6f268a210d4 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 15:15:56 +0100 Subject: [PATCH 07/10] Re-grumpf --- src/collective/documentgenerator/migrations/migrate_to_16.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py index fd4f1a42..39710fa2 100644 --- a/src/collective/documentgenerator/migrations/migrate_to_16.py +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -51,7 +51,6 @@ def _clean_expr(self): for info in infos: # collective.documentgenerator < 3.30 from which we use appy.pod S&R # XXX to be removed when using collective.documentgenerator >= 3.30 - import ipdb; ipdb.set_trace() if hasattr(info, 'pod_expr'): data[pt_path_and_title].append("---- " + info.pod_expr) data[pt_path_and_title].append("++++ " + info.new_pod_expr) From 2826c0a2eb79e5cecb81e2c7cd3c7246821b75f5 Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Tue, 24 Mar 2026 16:22:20 +0100 Subject: [PATCH 08/10] Pinned cryptography related packages --- buildout.d/plone-4.3.x.cfg | 7 +++++++ buildout.d/plone-5.1.x.cfg | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/buildout.d/plone-4.3.x.cfg b/buildout.d/plone-4.3.x.cfg index ae24f89b..55f843a1 100644 --- a/buildout.d/plone-4.3.x.cfg +++ b/buildout.d/plone-4.3.x.cfg @@ -10,3 +10,10 @@ backports.functools-lru-cache = 1.5 zc.lockfile = 2.0 zope.configuration = 3.8.1 shortuuid = 0.5.0 +cryptography = 3.3.2 +psutil = 6.1.1 +shortuuid = 0.5.0 +wcwidth = 0.2.5 +PyJWT = 1.7.1 +cffi = 1.15.1 +pycparser = 2.21 diff --git a/buildout.d/plone-5.1.x.cfg b/buildout.d/plone-5.1.x.cfg index dfad341a..b8bdd5a5 100644 --- a/buildout.d/plone-5.1.x.cfg +++ b/buildout.d/plone-5.1.x.cfg @@ -5,3 +5,10 @@ extends = [versions] shortuuid = 0.5.0 +cryptography = 3.3.2 +psutil = 6.1.1 +shortuuid = 0.5.0 +wcwidth = 0.2.5 +PyJWT = 1.7.1 +cffi = 1.15.1 +pycparser = 2.21 From b9e7916f440a588e3bd0da60ad3d419ff93a7b5f Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Wed, 25 Mar 2026 09:19:21 +0100 Subject: [PATCH 09/10] Fixed migration to 16 that will clean existing POD templates --- .../migrations/migrate_to_16.py | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/collective/documentgenerator/migrations/migrate_to_16.py b/src/collective/documentgenerator/migrations/migrate_to_16.py index 39710fa2..a0893433 100644 --- a/src/collective/documentgenerator/migrations/migrate_to_16.py +++ b/src/collective/documentgenerator/migrations/migrate_to_16.py @@ -28,14 +28,15 @@ def _clean_expr(self): for brain in self.catalog(object_provides=IPODTemplate.__identifier__): pod_template = brain.getObject() with SearchAndReplacePODTemplates([pod_template]) as search_replace: + # banned for banned_expr in Compromiser.banned: - res = search_replace.replace(banned_expr, '++REPLACED++', is_regex=False) - if res: - results.append(res) - for underscored_regex in Compromiser.underscored: - res = search_replace.replace(underscored_regex.pattern, '++REPLACED++', is_regex=True) + res = search_replace.replace(Compromiser.getBannedRex().pattern, '++REPLACED++', is_regex=True) if res: results.append(res) + # underscored + res = search_replace.replace(Compromiser.underscored.pattern, '++REPLACED++', is_regex=True) + if res: + results.append(res) # format results and dump it in the Zope log # as clean as possible so it can be used to know what changed data = {} @@ -49,15 +50,9 @@ def _clean_expr(self): self.warnings.append('Replacements were done in POD template at %s' % pt_path_and_title) for info in infos: - # collective.documentgenerator < 3.30 from which we use appy.pod S&R - # XXX to be removed when using collective.documentgenerator >= 3.30 - if hasattr(info, 'pod_expr'): - data[pt_path_and_title].append("---- " + info.pod_expr) - data[pt_path_and_title].append("++++ " + info.new_pod_expr) - else: - line = repr(info).replace(' These changes were done:', '>>>'). \ - replace('\n\n', '\n').rstrip('\n') - data[pt_path_and_title].append(line) + line = repr(info).replace(' These changes were done:', '>>>'). \ + replace('\n\n', '\n').rstrip('\n') + data[pt_path_and_title].append(line) logger.info("REPLACEMENTS IN POD TEMPLATES") if not data: logger.info("=============================") @@ -67,12 +62,10 @@ def _clean_expr(self): ordered_data = OrderedDict(sorted(data.items())) output = ["============================="] for pt_path_and_title, infos in ordered_data.items(): - output.append('\n') output.append("POD template " + pt_path_and_title) output.append('-' * len("POD template " + pt_path_and_title)) for info in infos: output.append(info) - output.append('\n') # make sure we do not mix unicode and utf-8 fixed_output = [] for line in output: From 0d0ce2cc0ad7dcd91ad98298fcc68170a5fa66ec Mon Sep 17 00:00:00 2001 From: Gauthier Bastien Date: Thu, 26 Mar 2026 09:55:33 +0100 Subject: [PATCH 10/10] Close tag `