diff --git a/Products/CMFPlone/MigrationTool.py b/Products/CMFPlone/MigrationTool.py
index 349ca888d5..3d45d2bd9f 100644
--- a/Products/CMFPlone/MigrationTool.py
+++ b/Products/CMFPlone/MigrationTool.py
@@ -2,10 +2,13 @@
from AccessControl.class_init import InitializeClass
from AccessControl.requestmethod import postonly
from App.config import getConfiguration
+from collections.abc import Generator
+from contextlib import contextmanager
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as dist_version
from io import StringIO
from OFS.SimpleItem import SimpleItem
+from OFS.Traversable import Traversable
from plone.base.interfaces import IMigrationTool
from Products.CMFCore.permissions import ManagePortal
from Products.CMFCore.utils import getToolByName
@@ -13,9 +16,13 @@
from Products.CMFCore.utils import UniqueObject
from Products.CMFPlone.factory import _DEFAULT_PROFILE
from Products.CMFPlone.PloneBaseTool import PloneBaseTool
+from Products.GenericSetup.tool import SetupTool
from ZODB.POSException import ConflictError
+from zope.component import getUtility
+from zope.component.hooks import getSite
from zope.interface import implementer
-
+from zope.interface import Interface
+from typing import TypedDict
import logging
import sys
import transaction
@@ -25,6 +32,44 @@
_upgradePaths = {}
+def _pil_version() -> str:
+ """Return version of the image package being used."""
+ version = "unknown"
+ try:
+ version = dist_version("PIL")
+ except PackageNotFoundError:
+ try:
+ version = dist_version("PILwoTK")
+ except PackageNotFoundError:
+ try:
+ pillow_version = dist_version("Pillow")
+ version = f"{pillow_version} (Pillow)"
+ except PackageNotFoundError:
+ try:
+ import _imaging
+
+ _imaging # pyflakes
+ except ImportError:
+ pass
+ return version
+
+
+@contextmanager
+def get_logger(stream: StringIO) -> Generator[logging.Logger]:
+ """Setup logging during migration."""
+ handler = logging.StreamHandler(stream)
+ handler.setLevel(logging.DEBUG)
+ logger.addHandler(handler)
+ gslogger = logging.getLogger("GenericSetup")
+ gslogger.addHandler(handler)
+ try:
+ yield logger
+ finally:
+ # Remove new handler
+ logger.removeHandler(handler)
+ gslogger.removeHandler(handler)
+
+
class Addon:
"""A profile or product.
@@ -46,10 +91,10 @@ def __init__(self, profile_id=None, check_module=None):
self.profile_id = profile_id
self.check_module = check_module
- def __repr__(self):
+ def __repr__(self) -> str:
return f"<{self.__class__.__name__} profile {self.profile_id}>"
- def safe(self):
+ def safe(self) -> bool:
# Is this addon safe to upgrade?
# Is it safe to pass its profile id to
@@ -69,52 +114,96 @@ def safe(self):
__import__(self.check_module)
except ImportError:
logger.info(
- "Cannot import module %s. Ignoring %s", self.check_module, self
+ f"Cannot import module {self.check_module}. Ignoring {self}"
)
return False
return True
class AddonList(list):
- def upgrade_all(self, context):
+
+ def upgrade_all(self, context: Traversable) -> None:
setup = getToolByName(context, "portal_setup")
for addon in self:
if addon.safe():
setup.upgradeProfile(addon.profile_id, quiet=True)
-# List of upgradeable packages. Obvious items to add here, are all
-# core packages that actually have upgrade steps.
-# Good start is portal_setup.listProfilesWithUpgrades()
-# Please use 'check_module' for packages that are not direct dependencies
-# of Products.CMFPlone, but of the Plone package.
-ADDON_LIST = AddonList(
- [
- Addon(profile_id="Products.CMFEditions:CMFEditions"),
- Addon(
- profile_id="Products.CMFPlacefulWorkflow:CMFPlacefulWorkflow",
- check_module="Products.CMFPlacefulWorkflow",
- ),
- Addon(profile_id="Products.PlonePAS:PlonePAS"),
- Addon(profile_id="plone.app.caching:default", check_module="plone.app.caching"),
- Addon(profile_id="plone.app.contenttypes:default"),
- Addon(profile_id="plone.app.dexterity:default"),
- Addon(profile_id="plone.app.discussion:default"),
- Addon(profile_id="plone.app.event:default"),
- Addon(profile_id="plone.app.iterate:default", check_module="plone.app.iterate"),
- Addon(
- profile_id="plone.app.multilingual:default",
- check_module="plone.app.multilingual",
- ),
- Addon(profile_id="plone.app.querystring:default"),
- Addon(profile_id="plone.app.theming:default"),
- Addon(profile_id="plone.app.users:default"),
- Addon(profile_id="plone.restapi:default", check_module="plone.restapi"),
- Addon(profile_id="plone.session:default"),
- Addon(profile_id="plone.staticresources:default"),
- Addon(profile_id="plone.volto:default", check_module="plone.volto"),
- Addon(profile_id="plonetheme.barceloneta:default"),
- ]
+class IAddonList(Interface):
+ """Utility providing a list of add ons managed by the migration tool."""
+
+ addon_list: AddonList
+
+
+@implementer(IAddonList)
+class LocalAddonList:
+ # List of upgradeable packages. Obvious items to add here, are all
+ # core packages that actually have upgrade steps.
+ # Good start is portal_setup.listProfilesWithUpgrades()
+ # Please use 'check_module' for packages that are not direct dependencies
+ # of Products.CMFPlone, but of the Plone package
+ addon_list: AddonList = AddonList(
+ [
+ Addon(profile_id="Products.CMFEditions:CMFEditions"),
+ Addon(
+ profile_id="Products.CMFPlacefulWorkflow:CMFPlacefulWorkflow",
+ check_module="Products.CMFPlacefulWorkflow",
+ ),
+ Addon(profile_id="Products.PlonePAS:PlonePAS"),
+ Addon(
+ profile_id="plone.app.caching:default", check_module="plone.app.caching"
+ ),
+ Addon(profile_id="plone.app.contenttypes:default"),
+ Addon(profile_id="plone.app.dexterity:default"),
+ Addon(profile_id="plone.app.discussion:default"),
+ Addon(profile_id="plone.app.event:default"),
+ Addon(
+ profile_id="plone.app.iterate:default", check_module="plone.app.iterate"
+ ),
+ Addon(
+ profile_id="plone.app.multilingual:default",
+ check_module="plone.app.multilingual",
+ ),
+ Addon(profile_id="plone.app.querystring:default"),
+ Addon(profile_id="plone.app.theming:default"),
+ Addon(profile_id="plone.app.users:default"),
+ Addon(profile_id="plone.restapi:default", check_module="plone.restapi"),
+ Addon(profile_id="plone.session:default"),
+ Addon(profile_id="plone.staticresources:default"),
+ Addon(profile_id="plone.volto:default", check_module="plone.volto"),
+ Addon(profile_id="plonetheme.barceloneta:default"),
+ ]
+ )
+
+
+_DEFAULT_PACKAGE_NAME = "Products.CMFPlone"
+_DEFAULT_FRIENDLY_NAME = "Plone"
+
+
+class CoreVersionInformation(TypedDict):
+ name: str
+ package_name: str
+ package_version: str
+ instance_version: str
+ fs_version: str
+
+
+VersionInformation = TypedDict(
+ "VersionInformation",
+ {
+ "Python": str,
+ "Zope": str,
+ "Platform": str,
+ "CMFPlone": str,
+ "Plone": str,
+ "Plone Instance": str,
+ "Plone File System": str,
+ "CMF": str,
+ "Debug mode": str,
+ "PIL": str,
+ "core": CoreVersionInformation,
+ "packages": dict[str, str],
+ },
)
@@ -122,52 +211,47 @@ def upgrade_all(self, context):
class MigrationTool(PloneBaseTool, UniqueObject, SimpleItem):
"""Handles migrations between Plone releases"""
- id = "portal_migration"
- meta_type = "Plone Migration Tool"
- toolicon = "skins/plone_images/site_icon.png"
+ id: str = "portal_migration"
+ meta_type: str = "Plone Migration Tool"
+ toolicon: str = "skins/plone_images/site_icon.png"
- _profile = _DEFAULT_PROFILE
- _package_name = "Products.CMFPlone"
+ profile: str = _DEFAULT_PROFILE
+ package_name: str = _DEFAULT_PACKAGE_NAME
+ friendly_name: str = _DEFAULT_FRIENDLY_NAME
- manage_options = (
+ manage_options: tuple[dict[str, str], ...] = (
{"label": "Upgrade", "action": "../@@plone-upgrade"},
) + SimpleItem.manage_options
- _needRecatalog = 0
- _needUpdateRole = 0
+ _needRecatalog: int = 0
+ _needUpdateRole: int = 0
security = ClassSecurityInfo()
- security.declareProtected(ManagePortal, "getInstanceVersion")
-
- security.declareProtected(ManagePortal, "getBaseProfile")
-
- def getBaseProfile(self):
- """Get the base profile used for migrations"""
- return getattr(self, "_profile", _DEFAULT_PROFILE)
-
- security.declareProtected(ManagePortal, "setBaseProfile")
+ security.declareProtected(ManagePortal, "initializeTool")
- def setBaseProfile(self, profile):
- """Set the base profile used for migrations"""
- self._profile = profile
+ @property
+ def setup(self) -> SetupTool:
+ site = getSite()
+ return getToolByName(site, "portal_setup")
- security.declareProtected(ManagePortal, "getPackageName")
+ def initializeTool(self, profile: str, package_name: str, friendly_name: str = ""):
+ """Initialize the migration tool."""
+ self.profile = profile
+ self.package_name = package_name
+ self.friendly_name = friendly_name if friendly_name else package_name
- def getPackageName(self):
- """Get the package name used for migrations"""
- return getattr(self, "_package_name", "Products.CMFPlone")
+ @property
+ def addon_list(self) -> AddonList:
+ utility = getUtility(IAddonList, self.package_name)
+ return utility.addon_list
- security.declareProtected(ManagePortal, "setPackageName")
-
- def setPackageName(self, package_name):
- """Set the package name used for migrations"""
- self._package_name = package_name
+ security.declareProtected(ManagePortal, "getInstanceVersion")
- def getInstanceVersion(self):
+ def getInstanceVersion(self) -> str:
# The version this instance of plone is on.
- setup = getToolByName(self, "portal_setup")
- profile = self.getBaseProfile()
+ setup = self.setup
+ profile = self.profile
version = setup.getLastVersionForProfile(profile)
if isinstance(version, tuple):
version = ".".join(version)
@@ -187,83 +271,89 @@ def getInstanceVersion(self):
_version = _version.replace("-", ".")
version = _version
else:
- version = setup.getVersionForProfile(_DEFAULT_PROFILE)
+ version = setup.getVersionForProfile(profile)
self.setInstanceVersion(version)
return version
security.declareProtected(ManagePortal, "setInstanceVersion")
- def setInstanceVersion(self, version):
+ def setInstanceVersion(self, version: str) -> None:
# The version this instance of plone is on.
- setup = getToolByName(self, "portal_setup")
- profile = self.getBaseProfile()
- setup.setLastVersionForProfile(profile, version)
+ setup = self.setup
+ setup.setLastVersionForProfile(self.profile, version)
self._version = False
security.declareProtected(ManagePortal, "getFileSystemVersion")
- def getFileSystemVersion(self):
+ def getFileSystemVersion(self) -> str | None:
# The version this instance of plone is on.
- setup = getToolByName(self, "portal_setup")
- profile = self.getBaseProfile()
+ setup = self.setup
try:
- return setup.getVersionForProfile(profile)
+ return setup.getVersionForProfile(self.profile)
except KeyError:
pass
return None
security.declareProtected(ManagePortal, "getSoftwareVersion")
- def getSoftwareVersion(self):
+ def getSoftwareVersion(self) -> str:
# The software version.
- package_name = self.getPackageName()
try:
- return dist_version(package_name)
+ return dist_version(self.package_name)
except PackageNotFoundError:
# Fall back to CMFPlone for backward compatibility
- return dist_version("Products.CMFPlone")
+ return dist_version(_DEFAULT_PACKAGE_NAME)
security.declareProtected(ManagePortal, "needUpgrading")
- def needUpgrading(self):
+ def needUpgrading(self) -> bool:
# Need upgrading?
return self.getInstanceVersion() != self.getFileSystemVersion()
security.declareProtected(ManagePortal, "coreVersions")
- def coreVersions(self):
+ def coreVersions(self) -> VersionInformation:
# Useful core information.
- vars = {}
- vars["Zope"] = dist_version("Zope")
- vars["Python"] = sys.version
- vars["Platform"] = sys.platform
- vars["Plone"] = dist_version("Products.CMFPlone")
- vars["Plone Instance"] = self.getInstanceVersion()
- vars["Plone File System"] = self.getFileSystemVersion()
- vars["CMF"] = dist_version("Products.CMFCore")
- vars["Debug mode"] = getConfiguration().debug_mode and "Yes" or "No"
- try:
- vars["PIL"] = dist_version("PIL")
- except PackageNotFoundError:
+ plone_version = dist_version("Products.CMFPlone")
+ instance_version = self.getInstanceVersion()
+ fs_version = self.getFileSystemVersion()
+ vars = {
+ "Python": sys.version,
+ "Zope": dist_version("Zope"),
+ "Platform": sys.platform,
+ "CMFPlone": plone_version,
+ "Plone": plone_version,
+ "Plone Instance": instance_version,
+ "Plone File System": fs_version,
+ "CMF": dist_version("Products.CMFCore"),
+ "Debug mode": "Yes" if getConfiguration().debug_mode else "No",
+ "PIL": _pil_version(),
+ "core": {
+ "name": self.friendly_name,
+ "package_name": self.package_name,
+ "package_version": self.getSoftwareVersion(),
+ "instance_version": instance_version,
+ "fs_version": fs_version,
+ },
+ "packages": {},
+ }
+ additional_packages = (
+ "plone.classicui",
+ "plone.distribution",
+ "plone.exportimport",
+ "plone.restapi",
+ "plone.volto",
+ )
+ for package_name in additional_packages:
try:
- vars["PIL"] = dist_version("PILwoTK")
+ vars["packages"][package_name] = dist_version(package_name)
except PackageNotFoundError:
- try:
- vars["PIL"] = "%s (Pillow)" % dist_version("Pillow")
- except PackageNotFoundError:
- try:
- import _imaging
-
- _imaging # pyflakes
- vars["PIL"] = "unknown"
- except ImportError:
- pass
-
+ pass
return vars
security.declareProtected(ManagePortal, "coreVersionsList")
- def coreVersionsList(self):
+ def coreVersionsList(self) -> list[str | dict | CoreVersionInformation]:
# Useful core information.
res = self.coreVersions().items()
res.sort()
@@ -271,36 +361,30 @@ def coreVersionsList(self):
security.declareProtected(ManagePortal, "needUpdateRole")
- def needUpdateRole(self):
+ def needUpdateRole(self) -> bool:
# Do roles need to be updated?
- return self._needUpdateRole
+ return bool(self._needUpdateRole)
security.declareProtected(ManagePortal, "needRecatalog")
- def needRecatalog(self):
+ def needRecatalog(self) -> bool:
# Does this thing now need recataloging?
- return self._needRecatalog
+ return bool(self._needRecatalog)
security.declareProtected(ManagePortal, "listUpgrades")
- def listUpgrades(self):
+ def listUpgrades(self) -> list:
# List available upgrade steps for our default profile.
# Do not include upgrade steps for too new versions:
# using a newer plone.app.upgrade version should not give problems.
- setup = getToolByName(self, "portal_setup")
- profile = self.getBaseProfile()
+ setup = self.setup
fs_version = self.getFileSystemVersion()
- upgrades = setup.listUpgrades(profile, dest=fs_version)
+ upgrades = setup.listUpgrades(self.profile, dest=fs_version)
return upgrades
- security.declareProtected(ManagePortal, "upgrade")
-
- def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True):
- # Perform the upgrade.
- setup = getToolByName(self, "portal_setup")
+ security.declareProtected(ManagePortal, "list_steps")
- # This sets the profile version if it wasn't set yet
- version = self.getInstanceVersion()
+ def list_steps(self) -> list:
upgrades = self.listUpgrades()
steps = []
for u in upgrades:
@@ -308,37 +392,84 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True):
steps.extend(u)
else:
steps.append(u)
+ return steps
+ def _upgrade_run_steps(
+ self, steps: list, logger: logging.Logger, swallow_errors: bool
+ ) -> None:
+ setup = self.setup
+ for step in steps:
+ try:
+ step_title = step["title"]
+ step["step"].doStep(setup)
+ setup.setLastVersionForProfile(self.profile, step["dest"])
+ logger.info(f"Ran upgrade step: {step_title}")
+ except (ConflictError, KeyboardInterrupt):
+ raise
+ except Exception:
+ logger.error("Upgrade aborted. Error:\n", exc_info=True)
+
+ if not swallow_errors:
+ raise
+ else:
+ # abort transaction to safe the zodb
+ transaction.abort()
+ break
+
+ def _upgrade_recatalog(self, logger: logging.Logger, swallow_errors: bool) -> None:
+ if not self.needRecatalog():
+ return
+ logger.info("Recatalog needed. This may take a while...")
try:
- stream = StringIO()
- handler = logging.StreamHandler(stream)
- handler.setLevel(logging.DEBUG)
- logger.addHandler(handler)
- gslogger = logging.getLogger("GenericSetup")
- gslogger.addHandler(handler)
+ catalog = self.portal_catalog
+ # Reduce threshold for the reindex run
+ old_threshold = catalog.threshold
+ pg_threshold = getattr(catalog, "pgthreshold", 0)
+ catalog.pgthreshold = 300
+ catalog.threshold = 2000
+ catalog.refreshCatalog(clear=1)
+ catalog.threshold = old_threshold
+ catalog.pgthreshold = pg_threshold
+ self._needRecatalog = 0
+ except (ConflictError, KeyboardInterrupt):
+ raise
+ except Exception:
+ logger.error(
+ "Exception was thrown while cataloging:\n",
+ exc_info=True,
+ )
+ if not swallow_errors:
+ raise
+
+ def _upgrade_roles(self, logger: logging.Logger, swallow_errors: bool) -> None:
+ if self.needUpdateRole():
+ logger.info("Role update needed. This may take a while...")
+ try:
+ self.portal_workflow.updateRoleMappings()
+ self._needUpdateRole = 0
+ except (ConflictError, KeyboardInterrupt):
+ raise
+ except Exception:
+ logger.error(
+ "Exception was thrown while updating role mappings",
+ exc_info=True,
+ )
+ if not swallow_errors:
+ raise
+ security.declareProtected(ManagePortal, "upgrade")
+
+ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True) -> str:
+ # This sets the profile version if it wasn't set yet
+ version = self.getInstanceVersion()
+ steps = self.list_steps()
+ stream = StringIO()
+ with get_logger(stream) as logger:
if dry_run:
logger.info("Dry run selected.")
- logger.info("Starting the migration from version: %s" % version)
-
- for step in steps:
- try:
- step["step"].doStep(setup)
- setup.setLastVersionForProfile(_DEFAULT_PROFILE, step["dest"])
- logger.info("Ran upgrade step: %s" % step["title"])
- except (ConflictError, KeyboardInterrupt):
- raise
- except Exception:
- logger.error("Upgrade aborted. Error:\n", exc_info=True)
-
- if not swallow_errors:
- raise
- else:
- # abort transaction to safe the zodb
- transaction.abort()
- break
-
+ logger.info(f"Starting the migration from version: {version}")
+ self._upgrade_run_steps(steps, logger, swallow_errors)
logger.info("End of upgrade path, main migration has finished.")
if self.needUpgrading():
@@ -346,46 +477,12 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True):
logger.error("Migration has failed")
else:
logger.info("Starting upgrade of core addons.")
- ADDON_LIST.upgrade_all(self)
+ self.addon_list.upgrade_all(self)
logger.info("Done upgrading core addons.")
# do this once all the changes have been done
- if self.needRecatalog():
- logger.info("Recatalog needed. This may take a while...")
- try:
- catalog = self.portal_catalog
- # Reduce threshold for the reindex run
- old_threshold = catalog.threshold
- pg_threshold = getattr(catalog, "pgthreshold", 0)
- catalog.pgthreshold = 300
- catalog.threshold = 2000
- catalog.refreshCatalog(clear=1)
- catalog.threshold = old_threshold
- catalog.pgthreshold = pg_threshold
- self._needRecatalog = 0
- except (ConflictError, KeyboardInterrupt):
- raise
- except Exception:
- logger.error(
- "Exception was thrown while cataloging:" "\n", exc_info=True
- )
- if not swallow_errors:
- raise
-
- if self.needUpdateRole():
- logger.info("Role update needed. This may take a while...")
- try:
- self.portal_workflow.updateRoleMappings()
- self._needUpdateRole = 0
- except (ConflictError, KeyboardInterrupt):
- raise
- except Exception:
- logger.error(
- "Exception was thrown while updating " "role mappings",
- exc_info=True,
- )
- if not swallow_errors:
- raise
+ self._upgrade_recatalog(logger, swallow_errors=swallow_errors)
+ self._upgrade_roles(logger, swallow_errors=swallow_errors)
logger.info("Your Plone instance is now up-to-date.")
if dry_run:
@@ -394,10 +491,6 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True):
return stream.getvalue()
- finally:
- logger.removeHandler(handler)
- gslogger.removeHandler(handler)
-
upgrade = postonly(upgrade)
diff --git a/Products/CMFPlone/configure.zcml b/Products/CMFPlone/configure.zcml
index b0f6a96eb3..232df767bd 100644
--- a/Products/CMFPlone/configure.zcml
+++ b/Products/CMFPlone/configure.zcml
@@ -97,6 +97,12 @@
+
+
+