From bb725f19f0c74e636953f05b0bcb7e06ae79f35f Mon Sep 17 00:00:00 2001 From: Beth Breisnes Date: Tue, 30 Nov 2021 16:16:38 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A2=20Auto-publish=20installers=20with?= =?UTF-8?q?=20new=20Publish=20Installer=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metaci/build/models.py | 11 ++ metaci/build/tests/test_models.py | 36 +++++- .../migrations/0041_auto_20211123_2247.py | 18 +++ .../migrations/0043_merge_20211130_2313.py | 14 ++ metaci/plan/models.py | 3 +- metaci/release/tasks.py | 58 ++++++--- metaci/release/tests/test_tasks.py | 120 +++++++++++++++++- package.json | 2 +- 8 files changed, 235 insertions(+), 27 deletions(-) create mode 100644 metaci/plan/migrations/0041_auto_20211123_2247.py create mode 100644 metaci/plan/migrations/0043_merge_20211130_2313.py diff --git a/metaci/build/models.py b/metaci/build/models.py index f8114f482..2a1ddbb87 100644 --- a/metaci/build/models.py +++ b/metaci/build/models.py @@ -731,6 +731,17 @@ def _get_flow_options(self) -> dict: if push_time: task_options["start_time"] = push_time.isoformat() + if ( + self.build.plan.role == ("publish_installer") + and self.build.release + ): + try: + publish_date = self.build.release.production_push_date.isoformat() + except AttributeError: + raise + else: + options["publish_date"] = publish_date + return options def set_commit_status(self): diff --git a/metaci/build/tests/test_models.py b/metaci/build/tests/test_models.py index fa9d4a3ae..1cb801c21 100644 --- a/metaci/build/tests/test_models.py +++ b/metaci/build/tests/test_models.py @@ -270,14 +270,48 @@ def test_get_flow_options__push_production(self): build_flow.build.release = Release( repo=build_flow.build.repo, change_case_template=change_case_template, + version_number="1.0", ) - build_flow.build.release.version_number = "1.0" build_flow.build.release.save() options = build_flow._get_flow_options() assert options["push_all"]["version"] == "1.0" expected = f"{datetime.date.today().isoformat()}T21:00:00+00:00" assert options["push_all"]["start_time"] == expected + def test_get_flow_options__publish_installer(self): + build_flow = BuildFlowFactory() + build_flow.build.plan = PlanFactory( + role="publish_installer", change_traffic_control=True + ) + build_flow.build.plan.save() + build_flow.build.repo = RepositoryFactory( + default_implementation_steps=[ + { + "role": "publish_installer", + "duration": 10, + "push_time": 21, + "start_time": 8, + "start_date_offset": 0, + }, + ], + ) + build_flow.build.repo.save() + planrepo = PlanRepositoryFactory( + plan=build_flow.build.plan, repo=build_flow.build.repo + ) + planrepo.save() + change_case_template = ChangeCaseTemplate() + change_case_template.save() + publish_date = datetime.date.today() + datetime.timedelta(days=6) + build_flow.build.release = Release( + repo=build_flow.build.repo, + change_case_template=change_case_template, + production_push_date=publish_date, + version_number="1.0", + ) + options = build_flow._get_flow_options() + assert options["publish_date"] == publish_date.isoformat() + def detach_logger(model): for handler in model.logger.handlers: diff --git a/metaci/plan/migrations/0041_auto_20211123_2247.py b/metaci/plan/migrations/0041_auto_20211123_2247.py new file mode 100644 index 000000000..f01bceeda --- /dev/null +++ b/metaci/plan/migrations/0041_auto_20211123_2247.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.13 on 2021-11-23 22:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('plan', '0040_plan_commit_status_regex'), + ] + + operations = [ + migrations.AlterField( + model_name='plan', + name='role', + field=models.CharField(choices=[('beta_release', 'Beta Release'), ('beta_test', 'Beta Test'), ('deploy', 'Deployment'), ('feature', 'Feature Test'), ('feature_robot', 'Feature Test Robot'), ('publish_installer', 'Publish Installer'), ('other', 'Other'), ('push_sandbox', 'Push Sandbox'), ('push_production', 'Push Production'), ('qa', 'QA Org'), ('release_deploy', 'Release Deploy'), ('release', 'Release'), ('release_test', 'Release Test'), ('scratch', 'Scratch Org')], max_length=17), + ), + ] diff --git a/metaci/plan/migrations/0043_merge_20211130_2313.py b/metaci/plan/migrations/0043_merge_20211130_2313.py new file mode 100644 index 000000000..accbacd99 --- /dev/null +++ b/metaci/plan/migrations/0043_merge_20211130_2313.py @@ -0,0 +1,14 @@ +# Generated by Django 3.1.13 on 2021-11-30 23:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plan', '0042_auto_20211117_0113'), + ('plan', '0041_auto_20211123_2247'), + ] + + operations = [ + ] diff --git a/metaci/plan/models.py b/metaci/plan/models.py index ab8d22e6e..dd300a20a 100644 --- a/metaci/plan/models.py +++ b/metaci/plan/models.py @@ -24,6 +24,7 @@ ("deploy", "Deployment"), ("feature", "Feature Test"), ("feature_robot", "Feature Test Robot"), + ("publish_installer", "Publish Installer"), ("other", "Other"), ("push_sandbox", "Push Sandbox"), ("push_production", "Push Production"), @@ -83,7 +84,7 @@ class Plan(models.Model): through_fields=("plan", "repo"), ) trigger = models.CharField(max_length=8, choices=TRIGGER_TYPES) - role = models.CharField(max_length=16, choices=BUILD_ROLES) + role = models.CharField(max_length=17, choices=BUILD_ROLES) queue = models.CharField(max_length=16, choices=QUEUES, default="default") regex = models.CharField(max_length=255, null=True, blank=True) commit_status_regex = models.CharField( diff --git a/metaci/release/tasks.py b/metaci/release/tasks.py index ed239681e..7ac622263 100644 --- a/metaci/release/tasks.py +++ b/metaci/release/tasks.py @@ -1,12 +1,12 @@ from collections import defaultdict from datetime import datetime, timezone -import json -import logging -from django.dispatch.dispatcher import receiver -from typing import Iterable, List, Optional +from typing import DefaultDict, List, Optional +from cumulusci.core.dependencies.dependencies import GitHubDynamicDependency +from cumulusci.core.github import get_github_api_for_repo +from cumulusci.utils.git import split_repo_url from django.conf import settings -from django.db.models.query import QuerySet, Q +from django.db.models.query import QuerySet from django.db.models.signals import post_delete, post_save from django.dispatch.dispatcher import receiver from django.urls import reverse @@ -16,19 +16,12 @@ from metaci.build.models import BUILD_STATUSES, Build from metaci.cumulusci.keychain import GitHubSettingsKeychain -from metaci.plan.models import PlanRepository +from metaci.plan.models import PlanRepository, Plan from metaci.release.models import Release, ReleaseCohort from metaci.repository.models import Repository - -from collections import defaultdict -from typing import DefaultDict, List - -from cumulusci.core.dependencies.dependencies import ( - GitHubDynamicDependency, +from cumulusci.core.dependencies.github import ( + get_remote_project_config, ) -from cumulusci.core.dependencies.resolvers import DependencyResolutionStrategy -from cumulusci.core.github import get_github_api_for_repo -from cumulusci.utils.git import split_repo_url class DependencyGraphError(Exception): @@ -174,10 +167,6 @@ def get_repo_from_url(self, url: str) -> Optional[GitHubRepository]: GitHubSettingsKeychain(), owner, name ).repository(owner, name) - @property - def logger(self): - return logging.getLogger(__name__) - def get_dependency_graph( releases: List[Release], @@ -272,9 +261,10 @@ def execute_active_release_cohorts(): for rc in ReleaseCohort.objects.filter( status=ReleaseCohort.STATUS.approved, dependency_graph__isnull=True ): - print("I found {rc}") create_dependency_tree(rc) + publish_installer_plans = Plan.objects.filter(role="publish_installer", active=True) + # Next, identify in-progress Release Cohorts that have reached a successful conclusion. # Release Cohorts whose component Releases fail are updated to a failure state by Release automation. for rc in ReleaseCohort.objects.filter(status=ReleaseCohort.STATUS.active).exclude( @@ -283,6 +273,9 @@ def execute_active_release_cohorts(): rc.status = ReleaseCohort.STATUS.completed rc.save() + if publish_installer_plans.count() == 1: + run_publish_installer_plans(rc, publish_installer_plans.first()) + # Next, identify in-progress Release Cohorts that need to be advanced. for rc in ReleaseCohort.objects.filter( status=ReleaseCohort.STATUS.active @@ -296,6 +289,31 @@ def execute_active_release_cohorts(): execute_active_release_cohorts_job = job(execute_active_release_cohorts) +# Run publish_installer for every repo that has a release in this cohort +# if that product has a metadeploy plan in its cumulusci.yml +def run_publish_installer_plans(rc: ReleaseCohort, publish_installer_plan: Plan): + for release in rc.releases.all(): + if release_has_plans(release): + build = Build( + repo=release.repo, + plan=publish_installer_plan, + commit=release.created_from_commit, + build_type="auto", + release=release, + release_relationship_type="automation", + ) + build.save() + + +def release_has_plans(release: Release) -> bool: + github_repo = release.repo.get_github_api() + config = get_remote_project_config( + github_repo, release.created_from_commit + ) # TODO: exception handling + print(config) + return len(config.get("plans", {})) > 0 + + def all_deps_satisfied( deps: List[str], graph: DefaultDict[str, List[str]], releases: List[Release] ) -> bool: diff --git a/metaci/release/tests/test_tasks.py b/metaci/release/tests/test_tasks.py index d81666f16..40bb864fe 100644 --- a/metaci/release/tests/test_tasks.py +++ b/metaci/release/tests/test_tasks.py @@ -27,6 +27,8 @@ all_deps_satisfied, create_dependency_tree, execute_active_release_cohorts, + run_publish_installer_plans, + release_has_plans, release_merge_freeze_if_safe, set_merge_freeze_status, ) @@ -578,9 +580,119 @@ def test_execute_active_release_cohorts__advances_release_cohorts( advance_releases_mock.assert_called_once_with(rc) -def test_get_dependency_graph(): - raise NotImplementedError +@pytest.mark.django_db +@unittest.mock.patch("metaci.release.tasks.run_publish_installer_plans") +@unittest.mock.patch("metaci.release.tasks.set_merge_freeze_status") +def test_execute_active_release_cohorts__run_publish_installer_plans( + smfs_mock, + run_publish_installer_plans_mock, +): + other_plan = PlanFactory(role="release", active=True) + publish_installer_plan = PlanFactory(role="publish_installer", active=True) + rc = ReleaseCohortFactory(status=ReleaseCohort.STATUS.active) + _ = ReleaseFactory( + repo__url="foo", release_cohort=rc, status=Release.STATUS.completed + ) + + execute_active_release_cohorts() + + run_publish_installer_plans_mock.assert_called_once_with(rc, publish_installer_plan) + + +@pytest.mark.django_db +@unittest.mock.patch("metaci.release.tasks.release_has_plans") +@unittest.mock.patch("metaci.release.tasks.set_merge_freeze_status") +def test_run_publish_installer_plans__with_cumulusci_yml_plans( + smfs_mock, + release_has_plans_mock, +): + publish_installer_plan = PlanFactory(role="publish_installer", active=True) + rc = ReleaseCohortFactory(status=ReleaseCohort.STATUS.active) + release = ReleaseFactory( + repo__url="foo", + release_cohort=rc, + status=Release.STATUS.completed, + created_from_commit="abc", + ) + release_has_plans_mock.return_value = True + + assert Build.objects.count() == 0 + + run_publish_installer_plans(rc, publish_installer_plan) + + assert Build.objects.count() == 1 + build = Build.objects.first() + assert build.release == release + assert build.plan == publish_installer_plan + + release_has_plans_mock.assert_called_once_with(release) + + +@pytest.mark.django_db +@unittest.mock.patch("metaci.release.tasks.release_has_plans") +@unittest.mock.patch("metaci.release.tasks.set_merge_freeze_status") +def test_run_publish_installer_plans__without_cumulusci_yml_plans( + smfs_mock, + release_has_plans_mock, +): + publish_installer_plan = PlanFactory(role="publish_installer", active=True) + rc = ReleaseCohortFactory(status=ReleaseCohort.STATUS.active) + release = ReleaseFactory( + repo__url="foo", + release_cohort=rc, + status=Release.STATUS.completed, + created_from_commit="abc", + ) + release_has_plans_mock.return_value = False + + assert Build.objects.count() == 0 + + run_publish_installer_plans(rc, publish_installer_plan) + + assert Build.objects.count() == 0 + + release_has_plans_mock.assert_called_once_with(release) + + +@pytest.mark.django_db +@unittest.mock.patch("metaci.release.tasks.get_remote_project_config") +@unittest.mock.patch("metaci.release.tasks.set_merge_freeze_status") +def test_release_has_plans__with_plans( + smfs_mock, + get_remote_project_config_mock, +): + rc = ReleaseCohortFactory(status=ReleaseCohort.STATUS.active) + repo = RepositoryFactory() + repo.get_github_api = unittest.mock.Mock() + release = ReleaseFactory( + repo=repo, + repo__url="foo", + release_cohort=rc, + status=Release.STATUS.completed, + created_from_commit="abc", + ) + get_remote_project_config_mock.return_value = {"plans": ["abc", "123"]} + + assert release_has_plans(release) + +@pytest.mark.django_db +@unittest.mock.patch("metaci.release.tasks.get_remote_project_config") +@unittest.mock.patch("metaci.release.tasks.set_merge_freeze_status") +def test_release_has_plans__without_plans( + smfs_mock, + get_remote_project_config_mock, +): + rc = ReleaseCohortFactory(status=ReleaseCohort.STATUS.active) + repo = RepositoryFactory() + repo.get_github_api = unittest.mock.Mock() + release = ReleaseFactory( + repo=repo, + repo__url="foo", + release_cohort=rc, + status=Release.STATUS.completed, + created_from_commit="abc", + ) + get_remote_project_config_mock.return_value = {} -def test_get_dependency_graph__duplicate_releases(): - raise NotImplementedError + assert not release_has_plans(release) diff --git a/package.json b/package.json index 598dd0301..ab3f6dd2b 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ }, "scripts": { "webpack:serve": "webpack serve --config webpack.dev.js", - "django:serve": "python manage.py runserver 0.0.0.0:8000", + "django:serve": "python manage.py runserver 0.0.0.0:${PORT:-8000}", "redis:clear": "redis-cli FLUSHALL", "rq:work": "python manage.py metaci_rqworker short", "rq:scheduler": "python manage.py metaci_rqscheduler --queue short",