From d8d0205f1d8345722e32672544fe6be91c91c2a0 Mon Sep 17 00:00:00 2001 From: dengyh Date: Thu, 15 Jan 2026 21:07:09 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=B5=81?= =?UTF-8?q?=E7=A8=8B,=E4=BB=BB=E5=8A=A1=E5=92=8C=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=9A=84=E8=BF=90=E8=90=A5=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20--story=3D130062520?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/unittest.yml | 14 +- bkflow/statistics/__init__.py | 0 bkflow/statistics/admin.py | 121 ++++++ bkflow/statistics/apps.py | 37 ++ bkflow/statistics/collectors/__init__.py | 28 ++ bkflow/statistics/collectors/base.py | 87 ++++ .../statistics/collectors/task_collector.py | 289 +++++++++++++ .../collectors/template_collector.py | 194 +++++++++ bkflow/statistics/conf.py | 79 ++++ bkflow/statistics/db_router.py | 70 ++++ bkflow/statistics/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/backfill_statistics.py | 197 +++++++++ bkflow/statistics/migrations/0001_initial.py | 317 +++++++++++++++ bkflow/statistics/migrations/__init__.py | 0 bkflow/statistics/models.py | 378 ++++++++++++++++++ bkflow/statistics/serializers.py | 122 ++++++ bkflow/statistics/signals/__init__.py | 22 + bkflow/statistics/signals/handlers.py | 116 ++++++ bkflow/statistics/tasks/__init__.py | 38 ++ bkflow/statistics/tasks/summary_tasks.py | 227 +++++++++++ bkflow/statistics/tasks/task_tasks.py | 77 ++++ bkflow/statistics/tasks/template_tasks.py | 52 +++ bkflow/statistics/urls.py | 30 ++ bkflow/statistics/views.py | 375 +++++++++++++++++ bkflow/urls.py | 11 + env.py | 26 ++ module_settings.py | 50 +++ scripts/run_all_unit_test.sh | 4 +- scripts/run_engine_unit_test.sh | 2 +- scripts/run_interface_unit_test.sh | 2 +- tests/engine/statistics/__init__.py | 0 tests/engine/statistics/test_collectors.py | 124 ++++++ tests/engine/statistics/test_conf.py | 73 ++++ tests/engine/statistics/test_models.py | 183 +++++++++ tests/engine/statistics/test_tasks.py | 138 +++++++ tests/interface/statistics/__init__.py | 0 tests/interface/statistics/test_collectors.py | 190 +++++++++ tests/interface/statistics/test_conf.py | 96 +++++ tests/interface/statistics/test_db_router.py | 120 ++++++ tests/interface/statistics/test_models.py | 152 +++++++ .../interface/statistics/test_serializers.py | 164 ++++++++ tests/interface/statistics/test_tasks.py | 156 ++++++++ tests/interface/statistics/test_views.py | 238 +++++++++++ tests/plugins/query/uniform_api/__init__.py | 18 + tests/plugins/query/uniform_api/test_utils.py | 158 ++++++++ tests/plugins/query/uniform_api/test_views.py | 165 ++++++++ 47 files changed, 4932 insertions(+), 8 deletions(-) create mode 100644 bkflow/statistics/__init__.py create mode 100644 bkflow/statistics/admin.py create mode 100644 bkflow/statistics/apps.py create mode 100644 bkflow/statistics/collectors/__init__.py create mode 100644 bkflow/statistics/collectors/base.py create mode 100644 bkflow/statistics/collectors/task_collector.py create mode 100644 bkflow/statistics/collectors/template_collector.py create mode 100644 bkflow/statistics/conf.py create mode 100644 bkflow/statistics/db_router.py create mode 100644 bkflow/statistics/management/__init__.py create mode 100644 bkflow/statistics/management/commands/__init__.py create mode 100644 bkflow/statistics/management/commands/backfill_statistics.py create mode 100644 bkflow/statistics/migrations/0001_initial.py create mode 100644 bkflow/statistics/migrations/__init__.py create mode 100644 bkflow/statistics/models.py create mode 100644 bkflow/statistics/serializers.py create mode 100644 bkflow/statistics/signals/__init__.py create mode 100644 bkflow/statistics/signals/handlers.py create mode 100644 bkflow/statistics/tasks/__init__.py create mode 100644 bkflow/statistics/tasks/summary_tasks.py create mode 100644 bkflow/statistics/tasks/task_tasks.py create mode 100644 bkflow/statistics/tasks/template_tasks.py create mode 100644 bkflow/statistics/urls.py create mode 100644 bkflow/statistics/views.py create mode 100644 tests/engine/statistics/__init__.py create mode 100644 tests/engine/statistics/test_collectors.py create mode 100644 tests/engine/statistics/test_conf.py create mode 100644 tests/engine/statistics/test_models.py create mode 100644 tests/engine/statistics/test_tasks.py create mode 100644 tests/interface/statistics/__init__.py create mode 100644 tests/interface/statistics/test_collectors.py create mode 100644 tests/interface/statistics/test_conf.py create mode 100644 tests/interface/statistics/test_db_router.py create mode 100644 tests/interface/statistics/test_models.py create mode 100644 tests/interface/statistics/test_serializers.py create mode 100644 tests/interface/statistics/test_tasks.py create mode 100644 tests/interface/statistics/test_views.py create mode 100644 tests/plugins/query/uniform_api/__init__.py create mode 100644 tests/plugins/query/uniform_api/test_utils.py create mode 100644 tests/plugins/query/uniform_api/test_views.py diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 3c73f3ba10..b517713501 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -16,9 +16,9 @@ jobs: python-version: [3.9] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -29,7 +29,7 @@ jobs: - name: Setup MySQL uses: shogo82148/actions-setup-mysql@v1 - id: setup-mysql # Add an ID to reference outputs + id: setup-mysql with: mysql-version: '8.0' user: 'test_user' @@ -61,3 +61,11 @@ jobs: sh scripts/run_interface_unit_test.sh echo "run engine unit test" sh scripts/run_engine_unit_test.sh + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: false + verbose: true diff --git a/bkflow/statistics/__init__.py b/bkflow/statistics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bkflow/statistics/admin.py b/bkflow/statistics/admin.py new file mode 100644 index 0000000000..f5fe630097 --- /dev/null +++ b/bkflow/statistics/admin.py @@ -0,0 +1,121 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.contrib import admin + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, + TemplateNodeStatistics, + TemplateStatistics, +) + + +@admin.register(TemplateNodeStatistics) +class TemplateNodeStatisticsAdmin(admin.ModelAdmin): + list_display = ["id", "template_id", "space_id", "component_code", "node_name", "is_remote", "created_at"] + list_filter = ["space_id", "is_remote", "is_sub"] + search_fields = ["component_code", "node_name", "template_id"] + readonly_fields = ["created_at"] + + +@admin.register(TemplateStatistics) +class TemplateStatisticsAdmin(admin.ModelAdmin): + list_display = [ + "id", + "template_id", + "space_id", + "template_name", + "atom_total", + "subprocess_total", + "is_enabled", + "updated_at", + ] + list_filter = ["space_id", "is_enabled"] + search_fields = ["template_name", "template_id"] + readonly_fields = ["created_at", "updated_at"] + + +@admin.register(TaskflowStatistics) +class TaskflowStatisticsAdmin(admin.ModelAdmin): + list_display = [ + "id", + "task_id", + "space_id", + "template_id", + "create_method", + "is_success", + "final_state", + "elapsed_time", + "create_time", + ] + list_filter = ["space_id", "create_method", "trigger_method", "is_success", "final_state"] + search_fields = ["task_id", "instance_id"] + readonly_fields = ["created_at", "updated_at"] + + +@admin.register(TaskflowExecutedNodeStatistics) +class TaskflowExecutedNodeStatisticsAdmin(admin.ModelAdmin): + list_display = [ + "id", + "task_id", + "node_id", + "component_code", + "status", + "state", + "elapsed_time", + "started_time", + ] + list_filter = ["space_id", "status", "state", "is_skip", "is_retry"] + search_fields = ["component_code", "node_name", "task_id"] + readonly_fields = ["created_at"] + + +@admin.register(DailyStatisticsSummary) +class DailyStatisticsSummaryAdmin(admin.ModelAdmin): + list_display = [ + "id", + "date", + "space_id", + "task_created_count", + "task_success_count", + "task_failed_count", + "avg_task_elapsed_time", + ] + list_filter = ["space_id", "date"] + readonly_fields = ["created_at", "updated_at"] + + +@admin.register(PluginExecutionSummary) +class PluginExecutionSummaryAdmin(admin.ModelAdmin): + list_display = [ + "id", + "period_type", + "period_start", + "space_id", + "component_code", + "execution_count", + "success_count", + "avg_elapsed_time", + ] + list_filter = ["space_id", "period_type", "is_remote"] + search_fields = ["component_code"] + readonly_fields = ["created_at", "updated_at"] diff --git a/bkflow/statistics/apps.py b/bkflow/statistics/apps.py new file mode 100644 index 0000000000..f33b63b283 --- /dev/null +++ b/bkflow/statistics/apps.py @@ -0,0 +1,37 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.apps import AppConfig + + +class AnalysisStatisticsConfig(AppConfig): + name = "bkflow.statistics" + verbose_name = "Statistics" + + def ready(self): + # 注册信号处理器 + try: + from bkflow.statistics.signals import register_statistics_signals + + register_statistics_signals() + except Exception as e: + import logging + + logger = logging.getLogger("root") + logger.warning(f"[AnalysisStatistics] Failed to register signals: {e}") diff --git a/bkflow/statistics/collectors/__init__.py b/bkflow/statistics/collectors/__init__.py new file mode 100644 index 0000000000..9f3cb4e2f9 --- /dev/null +++ b/bkflow/statistics/collectors/__init__.py @@ -0,0 +1,28 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from bkflow.statistics.collectors.base import BaseStatisticsCollector +from bkflow.statistics.collectors.task_collector import TaskStatisticsCollector +from bkflow.statistics.collectors.template_collector import TemplateStatisticsCollector + +__all__ = [ + "BaseStatisticsCollector", + "TemplateStatisticsCollector", + "TaskStatisticsCollector", +] diff --git a/bkflow/statistics/collectors/base.py b/bkflow/statistics/collectors/base.py new file mode 100644 index 0000000000..d5bf8c1afe --- /dev/null +++ b/bkflow/statistics/collectors/base.py @@ -0,0 +1,87 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging +from abc import ABC, abstractmethod +from typing import Tuple + +from bkflow.statistics.conf import StatisticsConfig + +logger = logging.getLogger("celery") + + +class BaseStatisticsCollector(ABC): + """统计采集器基类""" + + def __init__(self): + self.db_alias = StatisticsConfig.get_db_alias() + self.engine_id = StatisticsConfig.get_engine_id() + + @abstractmethod + def collect(self): + """执行采集""" + raise NotImplementedError + + @staticmethod + def count_pipeline_tree_nodes(pipeline_tree: dict) -> Tuple[int, int, int]: + """ + 统计 pipeline_tree 中的节点数量 + + Args: + pipeline_tree: pipeline 树结构 + + Returns: + tuple: (atom_total, subprocess_total, gateways_total) + """ + activities = pipeline_tree.get("activities", {}) + gateways = pipeline_tree.get("gateways", {}) + + atom_total = 0 + subprocess_total = 0 + + for act_id, act in activities.items(): + act_type = act.get("type", "") + if act_type == "ServiceActivity": + atom_total += 1 + elif act_type == "SubProcess": + subprocess_total += 1 + # 递归统计子流程内的节点 + sub_pipeline = act.get("pipeline", {}) + if sub_pipeline: + sub_atom, sub_subproc, _ = BaseStatisticsCollector.count_pipeline_tree_nodes(sub_pipeline) + atom_total += sub_atom + subprocess_total += sub_subproc + + gateways_total = len(gateways) + + return atom_total, subprocess_total, gateways_total + + @staticmethod + def parse_datetime(time_str): + """解析时间字符串""" + if not time_str: + return None + from django.utils.dateparse import parse_datetime + + # 处理多种可能的时间格式 + if isinstance(time_str, str): + # 替换空格分隔的时区格式 + time_str = time_str.replace(" +", "+").replace(" -", "-") + return parse_datetime(time_str) + return time_str diff --git a/bkflow/statistics/collectors/task_collector.py b/bkflow/statistics/collectors/task_collector.py new file mode 100644 index 0000000000..d01a8699b3 --- /dev/null +++ b/bkflow/statistics/collectors/task_collector.py @@ -0,0 +1,289 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging +from copy import deepcopy +from typing import List, Optional + +import ujson as json +from django.utils import timezone + +from bkflow.statistics.collectors.base import BaseStatisticsCollector +from bkflow.statistics.conf import StatisticsConfig +from bkflow.statistics.models import TaskflowExecutedNodeStatistics, TaskflowStatistics + +logger = logging.getLogger("celery") + + +class TaskStatisticsCollector(BaseStatisticsCollector): + """任务统计采集器""" + + def __init__(self, task_id: int = None, instance_id: str = None): + super().__init__() + self.task_id = task_id + self.instance_id = instance_id + self._task = None + + @property + def task(self): + """获取任务实例""" + if self._task is None: + try: + from bkflow.task.models import TaskInstance + + if self.task_id: + self._task = TaskInstance.objects.get(id=self.task_id) + elif self.instance_id: + self._task = TaskInstance.objects.get(instance_id=self.instance_id) + except Exception as e: + logger.warning(f"TaskInstance not found: task_id={self.task_id}, instance_id={self.instance_id}: {e}") + return self._task + + def collect(self): + """执行任务统计采集（创建时）""" + return self.collect_on_create() + + def collect_on_create(self): + """任务创建时采集""" + if not self.task: + return False + + # 检查是否需要排除 Mock 任务 + if self.task.create_method == "MOCK" and not StatisticsConfig.include_mock_tasks(): + logger.debug(f"Skip mock task statistics: task_id={self.task.id}") + return False + + try: + pipeline_tree = self.task.execution_data or {} + atom_total, subprocess_total, gateways_total = self.count_pipeline_tree_nodes(pipeline_tree) + + TaskflowStatistics.objects.using(self.db_alias).update_or_create( + task_id=self.task.id, + defaults={ + "instance_id": self.task.instance_id, + "template_id": self.task.template_id, + "space_id": self.task.space_id, + "scope_type": self.task.scope_type, + "scope_value": self.task.scope_value, + "engine_id": self.engine_id, + "atom_total": atom_total, + "subprocess_total": subprocess_total, + "gateways_total": gateways_total, + "node_total": atom_total + subprocess_total + gateways_total + 2, # +2 for start/end + "creator": self.task.creator, + "executor": self.task.executor, + "create_time": self.task.create_time, + "create_method": self.task.create_method, + "trigger_method": getattr(self.task, "trigger_method", "manual"), + "is_started": self.task.is_started, + "is_finished": False, + "is_success": False, + }, + ) + return True + except Exception as e: + logger.exception(f"[TaskStatisticsCollector] collect_on_create error: {e}") + return False + + def collect_on_archive(self): + """任务归档时采集（完成/撤销）""" + if not self.task: + return False + + try: + # 1. 更新任务统计 + self._update_task_statistics() + + # 2. 采集节点执行统计 + self._collect_node_statistics() + + return True + except Exception as e: + logger.exception(f"[TaskStatisticsCollector] collect_on_archive error: {e}") + return False + + def _update_task_statistics(self): + """更新任务统计信息""" + is_success = self.task.is_finished and not self.task.is_revoked + final_state = "" + if self.task.is_finished: + final_state = "FINISHED" + elif self.task.is_revoked: + final_state = "REVOKED" + + elapsed_time = None + if self.task.start_time and self.task.finish_time: + elapsed_time = int((self.task.finish_time - self.task.start_time).total_seconds()) + + TaskflowStatistics.objects.using(self.db_alias).filter(task_id=self.task.id).update( + start_time=self.task.start_time, + finish_time=self.task.finish_time, + elapsed_time=elapsed_time, + is_started=self.task.is_started, + is_finished=self.task.is_finished, + is_success=is_success, + final_state=final_state, + executor=self.task.executor, + ) + + def _collect_node_statistics(self): + """采集节点执行统计""" + try: + from bamboo_engine import api as bamboo_engine_api + from pipeline.eri.runtime import BambooDjangoRuntime + + # 获取状态树 + runtime = BambooDjangoRuntime() + status_result = bamboo_engine_api.get_pipeline_states( + runtime=runtime, + root_id=self.task.instance_id, + ) + + if not status_result.result: + logger.error(f"get_pipeline_states failed: {status_result.message}") + return + + status_tree = status_result.data + pipeline_tree = self.task.execution_data or {} + + # 删除旧数据 + TaskflowExecutedNodeStatistics.objects.using(self.db_alias).filter(task_id=self.task.id).delete() + + # 采集节点数据 + executed_nodes = self._extract_executed_nodes(pipeline_tree, status_tree) + + if executed_nodes: + TaskflowExecutedNodeStatistics.objects.using(self.db_alias).bulk_create( + executed_nodes, + batch_size=100, + ) + + # 更新任务统计的节点计数 + executed_count = len([n for n in executed_nodes if not n.is_retry]) + failed_count = len([n for n in executed_nodes if not n.status and not n.is_retry]) + retry_count = len([n for n in executed_nodes if n.is_retry]) + + TaskflowStatistics.objects.using(self.db_alias).filter(task_id=self.task.id).update( + executed_node_count=executed_count, + failed_node_count=failed_count, + retry_node_count=retry_count, + ) + except ImportError: + logger.warning("bamboo_engine not available, skip node statistics collection") + except Exception as e: + logger.exception(f"[TaskStatisticsCollector] _collect_node_statistics error: {e}") + + def _extract_executed_nodes( + self, + pipeline_tree: dict, + status_tree: dict, + subprocess_stack: list = None, + is_sub: bool = False, + ) -> List[TaskflowExecutedNodeStatistics]: + """递归提取已执行节点""" + if subprocess_stack is None: + subprocess_stack = [] + + nodes = [] + activities = pipeline_tree.get("activities", {}) + children = status_tree.get("children", {}) + + for act_id, act in activities.items(): + if act_id not in children: + continue + + node_status = children[act_id] + act_type = act.get("type", "") + + if act_type == "ServiceActivity": + node = self._create_node_statistics(act, node_status, subprocess_stack, is_sub) + if node: + nodes.append(node) + + elif act_type == "SubProcess": + sub_pipeline = act.get("pipeline", {}) + sub_status = node_status.get("children", {}) + if sub_pipeline and sub_status: + new_stack = deepcopy(subprocess_stack) + new_stack.insert(0, act_id) + sub_nodes = self._extract_executed_nodes( + sub_pipeline, + {"children": sub_status}, + new_stack, + True, + ) + nodes.extend(sub_nodes) + + return nodes + + def _create_node_statistics( + self, + activity: dict, + status: dict, + subprocess_stack: list, + is_sub: bool, + ) -> Optional[TaskflowExecutedNodeStatistics]: + """创建节点统计记录""" + state = status.get("state", "") + if state not in ("FINISHED", "FAILED", "REVOKED", "SUSPENDED"): + return None + + component = activity.get("component", {}) + component_code = component.get("code", "") + version = component.get("version", "legacy") + is_remote = False + + if component_code == "remote_plugin": + inputs = component.get("inputs", {}) + component_code = inputs.get("plugin_code", {}).get("value", component_code) + version = inputs.get("plugin_version", {}).get("value", version) + is_remote = True + + started_time = self.parse_datetime(status.get("start_time")) + archived_time = self.parse_datetime(status.get("finish_time")) + elapsed_time = status.get("elapsed_time") + + return TaskflowExecutedNodeStatistics( + component_code=component_code, + version=version, + is_remote=is_remote, + task_id=self.task.id, + instance_id=self.task.instance_id, + template_id=self.task.template_id, + space_id=self.task.space_id, + scope_type=self.task.scope_type, + scope_value=self.task.scope_value, + engine_id=self.engine_id, + node_id=activity.get("id", ""), + node_name=activity.get("name", ""), + template_node_id=activity.get("template_node_id", ""), + is_sub=is_sub, + subprocess_stack=json.dumps(subprocess_stack), + started_time=started_time or timezone.now(), + archived_time=archived_time, + elapsed_time=elapsed_time, + status=(state == "FINISHED"), + state=state, + is_skip=status.get("skip", False), + retry_count=status.get("retry", 0), + loop_count=status.get("loop", 1), + task_create_time=self.task.create_time, + task_start_time=self.task.start_time, + task_finish_time=self.task.finish_time, + ) diff --git a/bkflow/statistics/collectors/template_collector.py b/bkflow/statistics/collectors/template_collector.py new file mode 100644 index 0000000000..e8730dde40 --- /dev/null +++ b/bkflow/statistics/collectors/template_collector.py @@ -0,0 +1,194 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging +from copy import deepcopy +from typing import List + +import ujson as json +from django.db import transaction + +from bkflow.statistics.collectors.base import BaseStatisticsCollector +from bkflow.statistics.models import TemplateNodeStatistics, TemplateStatistics + +logger = logging.getLogger("celery") + + +class TemplateStatisticsCollector(BaseStatisticsCollector): + """模板统计采集器""" + + def __init__(self, template_id: int): + super().__init__() + self.template_id = template_id + self._template = None + + @property + def template(self): + """获取模板实例""" + if self._template is None: + try: + from bkflow.template.models import Template + + self._template = Template.objects.get(id=self.template_id) + except Exception as e: + logger.warning(f"Template {self.template_id} not found: {e}") + return self._template + + def collect(self): + """执行模板统计采集""" + if not self.template: + logger.warning(f"Template {self.template_id} not found, skip statistics collection") + return False + + try: + # 1. 采集模板节点统计 + self._collect_node_statistics() + + # 2. 采集模板整体统计 + self._collect_template_statistics() + + return True + except Exception as e: + logger.exception(f"[TemplateStatisticsCollector] template_id={self.template_id} error: {e}") + return False + + def _collect_node_statistics(self): + """采集模板节点统计""" + pipeline_tree = self.template.pipeline_tree or {} + + # 删除旧数据 + with transaction.atomic(using=self.db_alias): + TemplateNodeStatistics.objects.using(self.db_alias).filter(template_id=self.template_id).delete() + + # 收集节点数据 + component_list = self._collect_nodes( + pipeline_tree=pipeline_tree, + subprocess_stack=[], + is_sub=False, + ) + + # 批量创建 + if component_list: + TemplateNodeStatistics.objects.using(self.db_alias).bulk_create(component_list, batch_size=100) + + def _collect_nodes( + self, + pipeline_tree: dict, + subprocess_stack: list, + is_sub: bool, + ) -> List[TemplateNodeStatistics]: + """递归收集模板中的节点统计数据""" + component_list = [] + activities = pipeline_tree.get("activities", {}) + + for act_id, act in activities.items(): + act_type = act.get("type", "") + + if act_type == "ServiceActivity": + # 标准插件节点 + component = act.get("component", {}) + component_code = component.get("code", "") + component_version = component.get("version", "legacy") + + # 判断是否第三方插件 + is_remote = False + if component_code == "remote_plugin": + inputs = component.get("inputs", {}) + component_code = inputs.get("plugin_code", {}).get("value", component_code) + component_version = inputs.get("plugin_version", {}).get("value", component_version) + is_remote = True + + node_stat = TemplateNodeStatistics( + component_code=component_code, + version=component_version, + is_remote=is_remote, + template_id=self.template_id, + space_id=self.template.space_id, + scope_type=self.template.scope_type, + scope_value=self.template.scope_value, + node_id=act_id, + node_name=act.get("name", ""), + is_sub=is_sub, + subprocess_stack=json.dumps(subprocess_stack), + template_creator=self.template.creator, + template_create_time=self.template.create_at, + template_update_time=self.template.update_at, + ) + component_list.append(node_stat) + + elif act_type == "SubProcess": + # 子流程节点，递归处理 + sub_pipeline = act.get("pipeline", {}) + if sub_pipeline: + new_stack = deepcopy(subprocess_stack) + new_stack.insert(0, act_id) + sub_components = self._collect_nodes( + pipeline_tree=sub_pipeline, + subprocess_stack=new_stack, + is_sub=True, + ) + component_list.extend(sub_components) + + return component_list + + def _collect_template_statistics(self): + """采集模板整体统计""" + pipeline_tree = self.template.pipeline_tree or {} + + # 统计节点数量 + atom_total, subprocess_total, gateways_total = self.count_pipeline_tree_nodes(pipeline_tree) + + # 统计变量数量 + data = pipeline_tree.get("data", {}) + constants = pipeline_tree.get("constants", {}) + + input_count = 0 + output_count = 0 + + # 通过 constants 统计变量 + for key, const in constants.items(): + source_type = const.get("source_type", "") + if source_type == "component_outputs": + output_count += 1 + else: + input_count += 1 + + # 如果没有 constants，通过 data 统计 + if not constants: + input_count = len(data.get("inputs", {})) + output_count = len(data.get("outputs", [])) + + TemplateStatistics.objects.using(self.db_alias).update_or_create( + template_id=self.template_id, + defaults={ + "space_id": self.template.space_id, + "scope_type": self.template.scope_type, + "scope_value": self.template.scope_value, + "atom_total": atom_total, + "subprocess_total": subprocess_total, + "gateways_total": gateways_total, + "input_count": input_count, + "output_count": output_count, + "template_name": self.template.name, + "template_creator": self.template.creator, + "template_create_time": self.template.create_at, + "template_update_time": self.template.update_at, + "is_enabled": self.template.is_enabled, + }, + ) diff --git a/bkflow/statistics/conf.py b/bkflow/statistics/conf.py new file mode 100644 index 0000000000..a17837a534 --- /dev/null +++ b/bkflow/statistics/conf.py @@ -0,0 +1,79 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.conf import settings + +import env + + +class StatisticsConfig: + """统计配置管理""" + + @classmethod + def is_enabled(cls) -> bool: + """是否启用统计功能""" + return getattr(env, "STATISTICS_ENABLED", True) + + @classmethod + def get_engine_id(cls) -> str: + """获取当前 Engine 标识""" + bkflow_module = getattr(settings, "BKFLOW_MODULE", None) + if bkflow_module and hasattr(bkflow_module, "code"): + return bkflow_module.code + return getattr(env, "BKFLOW_MODULE_CODE", "default") + + @classmethod + def get_db_alias(cls) -> str: + """获取统计数据库别名""" + if "statistics" in settings.DATABASES: + return "statistics" + return "default" + + @classmethod + def should_collect_template_stats(cls) -> bool: + """是否采集模板统计（Interface 模块执行）""" + if not cls.is_enabled(): + return False + # 只在 Interface 模块或未指定模块类型时采集模板统计 + module_type = getattr(env, "BKFLOW_MODULE_TYPE", "") + return module_type in ("interface", "") + + @classmethod + def should_collect_task_stats(cls) -> bool: + """是否采集任务统计（Engine 模块执行）""" + if not cls.is_enabled(): + return False + # 只在 Engine 模块采集任务统计 + module_type = getattr(env, "BKFLOW_MODULE_TYPE", "") + return module_type == "engine" + + @classmethod + def include_mock_tasks(cls) -> bool: + """是否统计 Mock 任务""" + return getattr(env, "STATISTICS_INCLUDE_MOCK", False) + + @classmethod + def include_deleted_tasks(cls) -> bool: + """是否统计已删除任务""" + return getattr(env, "STATISTICS_INCLUDE_DELETED", False) + + @classmethod + def get_retention_days(cls) -> int: + """获取统计数据保留天数""" + return getattr(env, "STATISTICS_RETENTION_DAYS", 365) diff --git a/bkflow/statistics/db_router.py b/bkflow/statistics/db_router.py new file mode 100644 index 0000000000..dd0db6c003 --- /dev/null +++ b/bkflow/statistics/db_router.py @@ -0,0 +1,70 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + + +class StatisticsDBRouter: + """ + 统计模块数据库路由 + + 将 statistics app 的所有模型路由到 statistics 数据库 + """ + + APP_LABEL = "statistics" + DB_ALIAS = "statistics" + + def db_for_read(self, model, **hints): + """读操作路由""" + if model._meta.app_label == self.APP_LABEL: + return self._get_db_alias() + return None + + def db_for_write(self, model, **hints): + """写操作路由""" + if model._meta.app_label == self.APP_LABEL: + return self._get_db_alias() + return None + + def allow_relation(self, obj1, obj2, **hints): + """ + 允许同一数据库内的关联 + 统计表之间可以关联，但不与其他 app 的表关联 + """ + if obj1._meta.app_label == self.APP_LABEL and obj2._meta.app_label == self.APP_LABEL: + return True + # 统计表不与其他表建立外键关联 + if obj1._meta.app_label == self.APP_LABEL or obj2._meta.app_label == self.APP_LABEL: + return False + return None + + def allow_migrate(self, db, app_label, model_name=None, **hints): + """迁移路由""" + if app_label == self.APP_LABEL: + # 统计模型只在 statistics 数据库迁移 + return db == self._get_db_alias() + else: + # 其他模型不在 statistics 数据库迁移 + return db != self.DB_ALIAS + + def _get_db_alias(self): + """获取数据库别名，如果 statistics 数据库不存在则使用 default""" + from django.conf import settings + + if self.DB_ALIAS in settings.DATABASES: + return self.DB_ALIAS + return "default" diff --git a/bkflow/statistics/management/__init__.py b/bkflow/statistics/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bkflow/statistics/management/commands/__init__.py b/bkflow/statistics/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bkflow/statistics/management/commands/backfill_statistics.py b/bkflow/statistics/management/commands/backfill_statistics.py new file mode 100644 index 0000000000..acfc97e725 --- /dev/null +++ b/bkflow/statistics/management/commands/backfill_statistics.py @@ -0,0 +1,197 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging +from datetime import date, timedelta + +from django.core.management.base import BaseCommand + +from bkflow.statistics.conf import StatisticsConfig + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Backfill statistics data from existing templates and tasks" + + def add_arguments(self, parser): + parser.add_argument( + "--type", + choices=["template", "task", "summary", "all"], + default="all", + help="Type of data to backfill", + ) + parser.add_argument( + "--space-id", + type=int, + help="Limit to specific space ID", + ) + parser.add_argument( + "--days", + type=int, + default=30, + help="Number of days to backfill for tasks (default: 30)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=100, + help="Batch size for processing (default: 100)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Only show what would be done without making changes", + ) + + def handle(self, *args, **options): + if not StatisticsConfig.is_enabled(): + self.stderr.write(self.style.ERROR("Statistics is disabled. Set STATISTICS_ENABLED=true to enable.")) + return + + backfill_type = options["type"] + space_id = options.get("space_id") + days = options["days"] + batch_size = options["batch_size"] + dry_run = options["dry_run"] + + if dry_run: + self.stdout.write(self.style.WARNING("DRY RUN MODE - No changes will be made")) + + if backfill_type in ("template", "all"): + self._backfill_templates(space_id, batch_size, dry_run) + + if backfill_type in ("task", "all"): + self._backfill_tasks(space_id, days, batch_size, dry_run) + + if backfill_type in ("summary", "all"): + self._backfill_summaries(space_id, days, dry_run) + + self.stdout.write(self.style.SUCCESS("Backfill completed")) + + def _backfill_templates(self, space_id, batch_size, dry_run): + """回填模板统计数据""" + self.stdout.write("Backfilling template statistics...") + + try: + from bkflow.statistics.collectors import TemplateStatisticsCollector + from bkflow.template.models import Template + + queryset = Template.objects.all() + if space_id: + queryset = queryset.filter(space_id=space_id) + + total = queryset.count() + self.stdout.write(f"Found {total} templates to process") + + if dry_run: + return + + success = 0 + failed = 0 + + for template in queryset.iterator(): + try: + collector = TemplateStatisticsCollector(template_id=template.id) + if collector.collect(): + success += 1 + else: + failed += 1 + except Exception as e: + logger.exception(f"Failed to collect template {template.id}: {e}") + failed += 1 + + if (success + failed) % batch_size == 0: + self.stdout.write(f"Processed {success + failed}/{total} templates...") + + self.stdout.write(self.style.SUCCESS(f"Templates: {success} success, {failed} failed")) + + except ImportError: + self.stderr.write(self.style.WARNING("Template model not available")) + + def _backfill_tasks(self, space_id, days, batch_size, dry_run): + """回填任务统计数据""" + self.stdout.write(f"Backfilling task statistics (last {days} days)...") + + try: + from django.utils import timezone + + from bkflow.statistics.collectors import TaskStatisticsCollector + from bkflow.task.models import TaskInstance + + start_date = timezone.now() - timedelta(days=days) + + queryset = TaskInstance.objects.filter(create_time__gte=start_date) + if space_id: + queryset = queryset.filter(space_id=space_id) + + total = queryset.count() + self.stdout.write(f"Found {total} tasks to process") + + if dry_run: + return + + success = 0 + failed = 0 + + for task in queryset.iterator(): + try: + collector = TaskStatisticsCollector(task_id=task.id) + if collector.collect_on_create(): + success += 1 + # 如果任务已完成，也采集归档统计 + if task.is_finished or task.is_revoked: + collector.collect_on_archive() + else: + failed += 1 + except Exception as e: + logger.exception(f"Failed to collect task {task.id}: {e}") + failed += 1 + + if (success + failed) % batch_size == 0: + self.stdout.write(f"Processed {success + failed}/{total} tasks...") + + self.stdout.write(self.style.SUCCESS(f"Tasks: {success} success, {failed} failed")) + + except ImportError: + self.stderr.write(self.style.WARNING("TaskInstance model not available")) + + def _backfill_summaries(self, space_id, days, dry_run): + """回填汇总统计数据""" + self.stdout.write(f"Backfilling summary statistics (last {days} days)...") + + if dry_run: + self.stdout.write(f"Would generate daily summaries for {days} days") + return + + from bkflow.statistics.tasks.summary_tasks import ( + _generate_daily_summary, + _generate_plugin_summary, + ) + + for i in range(days): + summary_date = date.today() - timedelta(days=i + 1) + try: + _generate_daily_summary(summary_date) + _generate_plugin_summary("day", summary_date) + self.stdout.write(f"Generated summary for {summary_date}") + except Exception as e: + logger.exception(f"Failed to generate summary for {summary_date}: {e}") + + self.stdout.write(self.style.SUCCESS(f"Generated summaries for {days} days")) diff --git a/bkflow/statistics/migrations/0001_initial.py b/bkflow/statistics/migrations/0001_initial.py new file mode 100644 index 0000000000..5e2ef6a99a --- /dev/null +++ b/bkflow/statistics/migrations/0001_initial.py @@ -0,0 +1,317 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="TemplateNodeStatistics", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("component_code", models.CharField(db_index=True, max_length=255, verbose_name="组件编码")), + ("version", models.CharField(default="legacy", max_length=255, verbose_name="插件版本")), + ("is_remote", models.BooleanField(default=False, verbose_name="是否第三方插件")), + ("template_id", models.BigIntegerField(db_index=True, verbose_name="模板ID")), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ( + "scope_type", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="范围类型"), + ), + ( + "scope_value", + models.CharField(blank=True, db_index=True, max_length=255, null=True, verbose_name="范围值"), + ), + ("node_id", models.CharField(max_length=64, verbose_name="节点ID")), + ("node_name", models.CharField(blank=True, max_length=255, null=True, verbose_name="节点名称")), + ("is_sub", models.BooleanField(default=False, verbose_name="是否子流程引用")), + ("subprocess_stack", models.TextField(default="[]", verbose_name="子流程堆栈")), + ( + "template_creator", + models.CharField(blank=True, max_length=255, null=True, verbose_name="模板创建者"), + ), + ("template_create_time", models.DateTimeField(null=True, verbose_name="模板创建时间")), + ("template_update_time", models.DateTimeField(null=True, verbose_name="模板更新时间")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ], + options={ + "verbose_name": "模板节点统计", + "verbose_name_plural": "模板节点统计", + }, + ), + migrations.CreateModel( + name="TemplateStatistics", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("template_id", models.BigIntegerField(db_index=True, unique=True, verbose_name="模板ID")), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ( + "scope_type", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="范围类型"), + ), + ( + "scope_value", + models.CharField(blank=True, db_index=True, max_length=255, null=True, verbose_name="范围值"), + ), + ("atom_total", models.IntegerField(default=0, verbose_name="标准插件节点总数")), + ("subprocess_total", models.IntegerField(default=0, verbose_name="子流程节点总数")), + ("gateways_total", models.IntegerField(default=0, verbose_name="网关节点总数")), + ("input_count", models.IntegerField(default=0, verbose_name="输入变量数")), + ("output_count", models.IntegerField(default=0, verbose_name="输出变量数")), + ("template_name", models.CharField(blank=True, max_length=255, null=True, verbose_name="模板名称")), + ( + "template_creator", + models.CharField(blank=True, max_length=255, null=True, verbose_name="模板创建者"), + ), + ("template_create_time", models.DateTimeField(db_index=True, null=True, verbose_name="模板创建时间")), + ("template_update_time", models.DateTimeField(null=True, verbose_name="模板更新时间")), + ("is_enabled", models.BooleanField(default=True, verbose_name="是否启用")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="记录更新时间")), + ], + options={ + "verbose_name": "模板统计", + "verbose_name_plural": "模板统计", + }, + ), + migrations.CreateModel( + name="TaskflowStatistics", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("task_id", models.BigIntegerField(db_index=True, unique=True, verbose_name="任务ID")), + ("instance_id", models.CharField(db_index=True, max_length=64, verbose_name="Pipeline实例ID")), + ( + "template_id", + models.BigIntegerField(blank=True, db_index=True, null=True, verbose_name="关联模板ID"), + ), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ( + "scope_type", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="范围类型"), + ), + ( + "scope_value", + models.CharField(blank=True, db_index=True, max_length=255, null=True, verbose_name="范围值"), + ), + ( + "engine_id", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="Engine标识"), + ), + ("atom_total", models.IntegerField(default=0, verbose_name="标准插件节点总数")), + ("subprocess_total", models.IntegerField(default=0, verbose_name="子流程节点总数")), + ("gateways_total", models.IntegerField(default=0, verbose_name="网关节点总数")), + ("node_total", models.IntegerField(default=0, verbose_name="总节点数")), + ("executed_node_count", models.IntegerField(default=0, verbose_name="执行节点数")), + ("failed_node_count", models.IntegerField(default=0, verbose_name="失败节点数")), + ("retry_node_count", models.IntegerField(default=0, verbose_name="重试节点数")), + ("creator", models.CharField(blank=True, max_length=128, verbose_name="创建者")), + ("executor", models.CharField(blank=True, max_length=128, verbose_name="执行者")), + ("create_time", models.DateTimeField(db_index=True, verbose_name="创建时间")), + ("start_time", models.DateTimeField(blank=True, null=True, verbose_name="启动时间")), + ("finish_time", models.DateTimeField(blank=True, null=True, verbose_name="结束时间")), + ("elapsed_time", models.IntegerField(blank=True, null=True, verbose_name="执行耗时(秒)")), + ( + "create_method", + models.CharField(db_index=True, default="API", max_length=32, verbose_name="创建方式"), + ), + ( + "trigger_method", + models.CharField(db_index=True, default="manual", max_length=32, verbose_name="触发方式"), + ), + ("is_started", models.BooleanField(default=False, verbose_name="是否已启动")), + ("is_finished", models.BooleanField(default=False, verbose_name="是否已完成")), + ("is_success", models.BooleanField(db_index=True, default=False, verbose_name="是否成功")), + ("final_state", models.CharField(db_index=True, default="", max_length=32, verbose_name="最终状态")), + ( + "app_code", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="调用方应用"), + ), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="记录更新时间")), + ], + options={ + "verbose_name": "任务统计", + "verbose_name_plural": "任务统计", + }, + ), + migrations.CreateModel( + name="TaskflowExecutedNodeStatistics", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("component_code", models.CharField(db_index=True, max_length=255, verbose_name="组件编码")), + ("version", models.CharField(default="legacy", max_length=255, verbose_name="插件版本")), + ("is_remote", models.BooleanField(default=False, verbose_name="是否第三方插件")), + ("task_id", models.BigIntegerField(db_index=True, verbose_name="任务ID")), + ("instance_id", models.CharField(db_index=True, max_length=64, verbose_name="Pipeline实例ID")), + ("template_id", models.BigIntegerField(blank=True, db_index=True, null=True, verbose_name="关联模板ID")), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ("scope_type", models.CharField(blank=True, max_length=64, null=True, verbose_name="范围类型")), + ("scope_value", models.CharField(blank=True, max_length=255, null=True, verbose_name="范围值")), + ( + "engine_id", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="Engine标识"), + ), + ("node_id", models.CharField(db_index=True, max_length=64, verbose_name="节点ID")), + ("node_name", models.CharField(blank=True, max_length=255, null=True, verbose_name="节点名称")), + ("template_node_id", models.CharField(blank=True, max_length=64, null=True, verbose_name="模板节点ID")), + ("is_sub", models.BooleanField(default=False, verbose_name="是否子流程引用")), + ("subprocess_stack", models.TextField(default="[]", verbose_name="子流程堆栈")), + ("started_time", models.DateTimeField(db_index=True, verbose_name="节点执行开始时间")), + ("archived_time", models.DateTimeField(blank=True, null=True, verbose_name="节点执行结束时间")), + ("elapsed_time", models.IntegerField(blank=True, null=True, verbose_name="节点执行耗时(秒)")), + ("status", models.BooleanField(default=False, verbose_name="是否执行成功")), + ("state", models.CharField(db_index=True, default="", max_length=32, verbose_name="节点状态")), + ("is_skip", models.BooleanField(default=False, verbose_name="是否跳过")), + ("is_retry", models.BooleanField(default=False, verbose_name="是否重试记录")), + ("retry_count", models.IntegerField(default=0, verbose_name="重试次数")), + ("is_timeout", models.BooleanField(default=False, verbose_name="是否超时")), + ( + "error_code", + models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="错误码"), + ), + ("loop_count", models.IntegerField(default=1, verbose_name="循环次数")), + ("task_create_time", models.DateTimeField(db_index=True, verbose_name="任务创建时间")), + ("task_start_time", models.DateTimeField(blank=True, null=True, verbose_name="任务启动时间")), + ("task_finish_time", models.DateTimeField(blank=True, null=True, verbose_name="任务结束时间")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ], + options={ + "verbose_name": "节点执行统计", + "verbose_name_plural": "节点执行统计", + }, + ), + migrations.CreateModel( + name="DailyStatisticsSummary", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("date", models.DateField(db_index=True, verbose_name="统计日期")), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ("scope_type", models.CharField(blank=True, max_length=64, null=True, verbose_name="范围类型")), + ("scope_value", models.CharField(blank=True, max_length=255, null=True, verbose_name="范围值")), + ("task_created_count", models.IntegerField(default=0, verbose_name="创建任务数")), + ("task_started_count", models.IntegerField(default=0, verbose_name="启动任务数")), + ("task_finished_count", models.IntegerField(default=0, verbose_name="完成任务数")), + ("task_success_count", models.IntegerField(default=0, verbose_name="成功任务数")), + ("task_failed_count", models.IntegerField(default=0, verbose_name="失败任务数")), + ("task_revoked_count", models.IntegerField(default=0, verbose_name="撤销任务数")), + ("node_executed_count", models.IntegerField(default=0, verbose_name="执行节点数")), + ("node_success_count", models.IntegerField(default=0, verbose_name="成功节点数")), + ("node_failed_count", models.IntegerField(default=0, verbose_name="失败节点数")), + ("node_retry_count", models.IntegerField(default=0, verbose_name="重试节点数")), + ("avg_task_elapsed_time", models.FloatField(default=0, verbose_name="平均任务耗时(秒)")), + ("max_task_elapsed_time", models.IntegerField(default=0, verbose_name="最大任务耗时(秒)")), + ("total_task_elapsed_time", models.BigIntegerField(default=0, verbose_name="总任务耗时(秒)")), + ("template_created_count", models.IntegerField(default=0, verbose_name="创建模板数")), + ("template_updated_count", models.IntegerField(default=0, verbose_name="更新模板数")), + ("active_template_count", models.IntegerField(default=0, verbose_name="活跃模板数")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="记录更新时间")), + ], + options={ + "verbose_name": "每日统计汇总", + "verbose_name_plural": "每日统计汇总", + "unique_together": {("date", "space_id", "scope_type", "scope_value")}, + }, + ), + migrations.CreateModel( + name="PluginExecutionSummary", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("period_type", models.CharField(db_index=True, max_length=16, verbose_name="周期类型")), + ("period_start", models.DateField(db_index=True, verbose_name="周期开始日期")), + ("space_id", models.BigIntegerField(db_index=True, verbose_name="空间ID")), + ("component_code", models.CharField(db_index=True, max_length=255, verbose_name="组件编码")), + ("version", models.CharField(default="legacy", max_length=255, verbose_name="插件版本")), + ("is_remote", models.BooleanField(default=False, verbose_name="是否第三方插件")), + ("execution_count", models.IntegerField(default=0, verbose_name="执行次数")), + ("success_count", models.IntegerField(default=0, verbose_name="成功次数")), + ("failed_count", models.IntegerField(default=0, verbose_name="失败次数")), + ("retry_count", models.IntegerField(default=0, verbose_name="重试次数")), + ("timeout_count", models.IntegerField(default=0, verbose_name="超时次数")), + ("avg_elapsed_time", models.FloatField(default=0, verbose_name="平均耗时(秒)")), + ("max_elapsed_time", models.IntegerField(default=0, verbose_name="最大耗时(秒)")), + ("min_elapsed_time", models.IntegerField(default=0, verbose_name="最小耗时(秒)")), + ("p95_elapsed_time", models.IntegerField(default=0, verbose_name="P95耗时(秒)")), + ("template_reference_count", models.IntegerField(default=0, verbose_name="模板引用数")), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="记录创建时间")), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="记录更新时间")), + ], + options={ + "verbose_name": "插件执行汇总", + "verbose_name_plural": "插件执行汇总", + "unique_together": {("period_type", "period_start", "space_id", "component_code", "version")}, + }, + ), + migrations.AddIndex( + model_name="templatenodestatistics", + index=models.Index(fields=["space_id", "component_code"], name="stats_space_component_idx"), + ), + migrations.AddIndex( + model_name="templatenodestatistics", + index=models.Index(fields=["template_id", "node_id"], name="stats_tpl_node_idx"), + ), + migrations.AddIndex( + model_name="templatestatistics", + index=models.Index(fields=["space_id", "template_create_time"], name="stats_tpl_space_time_idx"), + ), + migrations.AddIndex( + model_name="taskflowstatistics", + index=models.Index(fields=["space_id", "create_time"], name="stats_task_space_time_idx"), + ), + migrations.AddIndex( + model_name="taskflowstatistics", + index=models.Index(fields=["template_id", "create_time"], name="stats_task_tpl_time_idx"), + ), + migrations.AddIndex( + model_name="taskflowstatistics", + index=models.Index(fields=["engine_id", "create_time"], name="stats_task_engine_time_idx"), + ), + migrations.AddIndex( + model_name="taskflowexecutednodestatistics", + index=models.Index(fields=["space_id", "component_code", "started_time"], name="stats_node_space_comp_idx"), + ), + migrations.AddIndex( + model_name="taskflowexecutednodestatistics", + index=models.Index(fields=["task_id", "node_id"], name="stats_node_task_idx"), + ), + migrations.AddIndex( + model_name="taskflowexecutednodestatistics", + index=models.Index(fields=["component_code", "status", "started_time"], name="stats_node_comp_status_idx"), + ), + migrations.AddIndex( + model_name="taskflowexecutednodestatistics", + index=models.Index(fields=["engine_id", "started_time"], name="stats_node_engine_time_idx"), + ), + migrations.AddIndex( + model_name="dailystatisticssummary", + index=models.Index(fields=["space_id", "date"], name="stats_daily_space_date_idx"), + ), + migrations.AddIndex( + model_name="pluginexecutionsummary", + index=models.Index( + fields=["space_id", "component_code", "period_start"], name="stats_plugin_space_comp_idx" + ), + ), + ] diff --git a/bkflow/statistics/migrations/__init__.py b/bkflow/statistics/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bkflow/statistics/models.py b/bkflow/statistics/models.py new file mode 100644 index 0000000000..7a4fea1700 --- /dev/null +++ b/bkflow/statistics/models.py @@ -0,0 +1,378 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.db import models + + +class StatisticsBaseModel(models.Model): + """ + 统计模型基类 + + 所有统计模型继承此基类，确保： + 1. 使用正确的 app_label（用于数据库路由） + 2. 统一的元信息配置 + """ + + class Meta: + abstract = True + app_label = "statistics" + + @classmethod + def using_db(cls): + """获取当前模型使用的数据库""" + from django.conf import settings + + return "statistics" if "statistics" in settings.DATABASES else "default" + + def save(self, *args, **kwargs): + """确保保存到正确的数据库""" + kwargs.setdefault("using", self.using_db()) + super().save(*args, **kwargs) + + def delete(self, *args, **kwargs): + """确保从正确的数据库删除""" + kwargs.setdefault("using", self.using_db()) + super().delete(*args, **kwargs) + + @classmethod + def objects_using_db(cls): + """获取使用正确数据库的 QuerySet""" + return cls.objects.using(cls.using_db()) + + +class TemplateNodeStatistics(StatisticsBaseModel): + """模板中标准插件节点的引用统计""" + + id = models.BigAutoField(primary_key=True) + + # 组件信息 + component_code = models.CharField("组件编码", max_length=255, db_index=True) + version = models.CharField("插件版本", max_length=255, default="legacy") + is_remote = models.BooleanField("是否第三方插件", default=False) + + # 模板关联 + template_id = models.BigIntegerField("模板ID", db_index=True) + + # 平台标识 + space_id = models.BigIntegerField("空间ID", db_index=True) + scope_type = models.CharField("范围类型", max_length=64, null=True, blank=True, db_index=True) + scope_value = models.CharField("范围值", max_length=255, null=True, blank=True, db_index=True) + + # 节点信息 + node_id = models.CharField("节点ID", max_length=64) + node_name = models.CharField("节点名称", max_length=255, null=True, blank=True) + is_sub = models.BooleanField("是否子流程引用", default=False) + subprocess_stack = models.TextField("子流程堆栈", default="[]") + + # 模板元信息 + template_creator = models.CharField("模板创建者", max_length=255, null=True, blank=True) + template_create_time = models.DateTimeField("模板创建时间", null=True) + template_update_time = models.DateTimeField("模板更新时间", null=True) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "模板节点统计" + verbose_name_plural = "模板节点统计" + indexes = [ + models.Index(fields=["space_id", "component_code"]), + models.Index(fields=["template_id", "node_id"]), + ] + + def __str__(self): + return f"{self.component_code}_{self.template_id}_{self.node_id}" + + +class TemplateStatistics(StatisticsBaseModel): + """模板维度的整体统计""" + + id = models.BigAutoField(primary_key=True) + + # 模板关联 + template_id = models.BigIntegerField("模板ID", db_index=True, unique=True) + + # 平台标识 + space_id = models.BigIntegerField("空间ID", db_index=True) + scope_type = models.CharField("范围类型", max_length=64, null=True, blank=True, db_index=True) + scope_value = models.CharField("范围值", max_length=255, null=True, blank=True, db_index=True) + + # 节点统计 + atom_total = models.IntegerField("标准插件节点总数", default=0) + subprocess_total = models.IntegerField("子流程节点总数", default=0) + gateways_total = models.IntegerField("网关节点总数", default=0) + + # 变量统计 + input_count = models.IntegerField("输入变量数", default=0) + output_count = models.IntegerField("输出变量数", default=0) + + # 模板元信息 + template_name = models.CharField("模板名称", max_length=255, null=True, blank=True) + template_creator = models.CharField("模板创建者", max_length=255, null=True, blank=True) + template_create_time = models.DateTimeField("模板创建时间", null=True, db_index=True) + template_update_time = models.DateTimeField("模板更新时间", null=True) + is_enabled = models.BooleanField("是否启用", default=True) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + updated_at = models.DateTimeField("记录更新时间", auto_now=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "模板统计" + verbose_name_plural = "模板统计" + indexes = [ + models.Index(fields=["space_id", "template_create_time"]), + ] + + def __str__(self): + return f"Template_{self.template_id}" + + +class TaskflowStatistics(StatisticsBaseModel): + """任务实例维度的统计""" + + id = models.BigAutoField(primary_key=True) + + # 任务关联 + task_id = models.BigIntegerField("任务ID", db_index=True, unique=True) + instance_id = models.CharField("Pipeline实例ID", max_length=64, db_index=True) + template_id = models.BigIntegerField("关联模板ID", null=True, blank=True, db_index=True) + + # 平台标识 + space_id = models.BigIntegerField("空间ID", db_index=True) + scope_type = models.CharField("范围类型", max_length=64, null=True, blank=True, db_index=True) + scope_value = models.CharField("范围值", max_length=255, null=True, blank=True, db_index=True) + + # Engine 标识（用于区分不同 Engine 模块写入的数据） + engine_id = models.CharField("Engine标识", max_length=64, null=True, blank=True, db_index=True) + + # 节点统计 + atom_total = models.IntegerField("标准插件节点总数", default=0) + subprocess_total = models.IntegerField("子流程节点总数", default=0) + gateways_total = models.IntegerField("网关节点总数", default=0) + node_total = models.IntegerField("总节点数", default=0) + executed_node_count = models.IntegerField("执行节点数", default=0) + failed_node_count = models.IntegerField("失败节点数", default=0) + retry_node_count = models.IntegerField("重试节点数", default=0) + + # 执行信息 + creator = models.CharField("创建者", max_length=128, blank=True) + executor = models.CharField("执行者", max_length=128, blank=True) + create_time = models.DateTimeField("创建时间", db_index=True) + start_time = models.DateTimeField("启动时间", null=True, blank=True) + finish_time = models.DateTimeField("结束时间", null=True, blank=True) + elapsed_time = models.IntegerField("执行耗时(秒)", null=True, blank=True) + + # 创建/触发方式 + create_method = models.CharField("创建方式", max_length=32, default="API", db_index=True) + trigger_method = models.CharField("触发方式", max_length=32, default="manual", db_index=True) + + # 执行状态 + is_started = models.BooleanField("是否已启动", default=False) + is_finished = models.BooleanField("是否已完成", default=False) + is_success = models.BooleanField("是否成功", default=False, db_index=True) + final_state = models.CharField("最终状态", max_length=32, default="", db_index=True) + + # 调用方信息 + app_code = models.CharField("调用方应用", max_length=64, null=True, blank=True, db_index=True) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + updated_at = models.DateTimeField("记录更新时间", auto_now=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "任务统计" + verbose_name_plural = "任务统计" + indexes = [ + models.Index(fields=["space_id", "create_time"]), + models.Index(fields=["template_id", "create_time"]), + models.Index(fields=["engine_id", "create_time"]), + ] + + def __str__(self): + return f"Task_{self.task_id}" + + +class TaskflowExecutedNodeStatistics(StatisticsBaseModel): + """任务执行过程中节点执行详情统计""" + + id = models.BigAutoField(primary_key=True) + + # 组件信息 + component_code = models.CharField("组件编码", max_length=255, db_index=True) + version = models.CharField("插件版本", max_length=255, default="legacy") + is_remote = models.BooleanField("是否第三方插件", default=False) + + # 任务关联 + task_id = models.BigIntegerField("任务ID", db_index=True) + instance_id = models.CharField("Pipeline实例ID", max_length=64, db_index=True) + template_id = models.BigIntegerField("关联模板ID", null=True, blank=True, db_index=True) + + # 平台标识 + space_id = models.BigIntegerField("空间ID", db_index=True) + scope_type = models.CharField("范围类型", max_length=64, null=True, blank=True) + scope_value = models.CharField("范围值", max_length=255, null=True, blank=True) + + # Engine 标识 + engine_id = models.CharField("Engine标识", max_length=64, null=True, blank=True, db_index=True) + + # 节点信息 + node_id = models.CharField("节点ID", max_length=64, db_index=True) + node_name = models.CharField("节点名称", max_length=255, null=True, blank=True) + template_node_id = models.CharField("模板节点ID", max_length=64, null=True, blank=True) + is_sub = models.BooleanField("是否子流程引用", default=False) + subprocess_stack = models.TextField("子流程堆栈", default="[]") + + # 执行信息 + started_time = models.DateTimeField("节点执行开始时间", db_index=True) + archived_time = models.DateTimeField("节点执行结束时间", null=True, blank=True) + elapsed_time = models.IntegerField("节点执行耗时(秒)", null=True, blank=True) + + # 执行状态 + status = models.BooleanField("是否执行成功", default=False) + state = models.CharField("节点状态", max_length=32, default="", db_index=True) + is_skip = models.BooleanField("是否跳过", default=False) + is_retry = models.BooleanField("是否重试记录", default=False) + retry_count = models.IntegerField("重试次数", default=0) + is_timeout = models.BooleanField("是否超时", default=False) + error_code = models.CharField("错误码", max_length=64, null=True, blank=True, db_index=True) + loop_count = models.IntegerField("循环次数", default=1) + + # 任务实例时间（冗余，便于统计） + task_create_time = models.DateTimeField("任务创建时间", db_index=True) + task_start_time = models.DateTimeField("任务启动时间", null=True, blank=True) + task_finish_time = models.DateTimeField("任务结束时间", null=True, blank=True) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "节点执行统计" + verbose_name_plural = "节点执行统计" + indexes = [ + models.Index(fields=["space_id", "component_code", "started_time"]), + models.Index(fields=["task_id", "node_id"]), + models.Index(fields=["component_code", "status", "started_time"]), + models.Index(fields=["engine_id", "started_time"]), + ] + + def __str__(self): + return f"{self.component_code}_{self.task_id}_{self.node_id}" + + +class DailyStatisticsSummary(StatisticsBaseModel): + """每日统计汇总（预计算）""" + + id = models.BigAutoField(primary_key=True) + + # 时间维度 + date = models.DateField("统计日期", db_index=True) + + # 空间维度 + space_id = models.BigIntegerField("空间ID", db_index=True) + scope_type = models.CharField("范围类型", max_length=64, null=True, blank=True) + scope_value = models.CharField("范围值", max_length=255, null=True, blank=True) + + # 任务统计 + task_created_count = models.IntegerField("创建任务数", default=0) + task_started_count = models.IntegerField("启动任务数", default=0) + task_finished_count = models.IntegerField("完成任务数", default=0) + task_success_count = models.IntegerField("成功任务数", default=0) + task_failed_count = models.IntegerField("失败任务数", default=0) + task_revoked_count = models.IntegerField("撤销任务数", default=0) + + # 节点统计 + node_executed_count = models.IntegerField("执行节点数", default=0) + node_success_count = models.IntegerField("成功节点数", default=0) + node_failed_count = models.IntegerField("失败节点数", default=0) + node_retry_count = models.IntegerField("重试节点数", default=0) + + # 耗时统计 + avg_task_elapsed_time = models.FloatField("平均任务耗时(秒)", default=0) + max_task_elapsed_time = models.IntegerField("最大任务耗时(秒)", default=0) + total_task_elapsed_time = models.BigIntegerField("总任务耗时(秒)", default=0) + + # 模板统计 + template_created_count = models.IntegerField("创建模板数", default=0) + template_updated_count = models.IntegerField("更新模板数", default=0) + active_template_count = models.IntegerField("活跃模板数", default=0) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + updated_at = models.DateTimeField("记录更新时间", auto_now=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "每日统计汇总" + verbose_name_plural = "每日统计汇总" + unique_together = ["date", "space_id", "scope_type", "scope_value"] + indexes = [ + models.Index(fields=["space_id", "date"]), + ] + + def __str__(self): + return f"DailySummary_{self.space_id}_{self.date}" + + +class PluginExecutionSummary(StatisticsBaseModel): + """插件执行汇总（按天/按周/按月）""" + + id = models.BigAutoField(primary_key=True) + + # 时间维度 + period_type = models.CharField("周期类型", max_length=16, db_index=True) # day/week/month + period_start = models.DateField("周期开始日期", db_index=True) + + # 空间维度 + space_id = models.BigIntegerField("空间ID", db_index=True) + + # 插件维度 + component_code = models.CharField("组件编码", max_length=255, db_index=True) + version = models.CharField("插件版本", max_length=255, default="legacy") + is_remote = models.BooleanField("是否第三方插件", default=False) + + # 执行统计 + execution_count = models.IntegerField("执行次数", default=0) + success_count = models.IntegerField("成功次数", default=0) + failed_count = models.IntegerField("失败次数", default=0) + retry_count = models.IntegerField("重试次数", default=0) + timeout_count = models.IntegerField("超时次数", default=0) + + # 耗时统计 + avg_elapsed_time = models.FloatField("平均耗时(秒)", default=0) + max_elapsed_time = models.IntegerField("最大耗时(秒)", default=0) + min_elapsed_time = models.IntegerField("最小耗时(秒)", default=0) + p95_elapsed_time = models.IntegerField("P95耗时(秒)", default=0) + + # 引用统计 + template_reference_count = models.IntegerField("模板引用数", default=0) + + # 记录时间 + created_at = models.DateTimeField("记录创建时间", auto_now_add=True) + updated_at = models.DateTimeField("记录更新时间", auto_now=True) + + class Meta(StatisticsBaseModel.Meta): + verbose_name = "插件执行汇总" + verbose_name_plural = "插件执行汇总" + unique_together = ["period_type", "period_start", "space_id", "component_code", "version"] + indexes = [ + models.Index(fields=["space_id", "component_code", "period_start"]), + ] + + def __str__(self): + return f"PluginSummary_{self.component_code}_{self.period_type}_{self.period_start}" diff --git a/bkflow/statistics/serializers.py b/bkflow/statistics/serializers.py new file mode 100644 index 0000000000..cbf25ff0ae --- /dev/null +++ b/bkflow/statistics/serializers.py @@ -0,0 +1,122 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from rest_framework import serializers + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, + TemplateNodeStatistics, + TemplateStatistics, +) + + +class TemplateNodeStatisticsSerializer(serializers.ModelSerializer): + class Meta: + model = TemplateNodeStatistics + fields = "__all__" + + +class TemplateStatisticsSerializer(serializers.ModelSerializer): + class Meta: + model = TemplateStatistics + fields = "__all__" + + +class TaskflowStatisticsSerializer(serializers.ModelSerializer): + class Meta: + model = TaskflowStatistics + fields = "__all__" + + +class TaskflowExecutedNodeStatisticsSerializer(serializers.ModelSerializer): + class Meta: + model = TaskflowExecutedNodeStatistics + fields = "__all__" + + +class DailyStatisticsSummarySerializer(serializers.ModelSerializer): + class Meta: + model = DailyStatisticsSummary + fields = "__all__" + + +class PluginExecutionSummarySerializer(serializers.ModelSerializer): + class Meta: + model = PluginExecutionSummary + fields = "__all__" + + +class StatisticsOverviewSerializer(serializers.Serializer): + """空间统计概览""" + + space_id = serializers.IntegerField() + scope_type = serializers.CharField(required=False, allow_null=True) + scope_value = serializers.CharField(required=False, allow_null=True) + + # 任务统计 + total_tasks = serializers.IntegerField() + success_tasks = serializers.IntegerField() + failed_tasks = serializers.IntegerField() + success_rate = serializers.FloatField() + + # 模板统计 + total_templates = serializers.IntegerField() + active_templates = serializers.IntegerField() + + # 节点统计 + total_nodes_executed = serializers.IntegerField() + node_success_rate = serializers.FloatField() + + # 耗时统计 + avg_task_elapsed_time = serializers.FloatField() + + +class TaskTrendSerializer(serializers.Serializer): + """任务趋势数据""" + + date = serializers.DateField() + task_created_count = serializers.IntegerField() + task_finished_count = serializers.IntegerField() + task_success_count = serializers.IntegerField() + success_rate = serializers.FloatField() + + +class PluginRankingSerializer(serializers.Serializer): + """插件排行数据""" + + component_code = serializers.CharField() + version = serializers.CharField() + execution_count = serializers.IntegerField() + success_count = serializers.IntegerField() + failed_count = serializers.IntegerField() + success_rate = serializers.FloatField() + avg_elapsed_time = serializers.FloatField() + + +class TemplateRankingSerializer(serializers.Serializer): + """模板排行数据""" + + template_id = serializers.IntegerField() + template_name = serializers.CharField() + task_count = serializers.IntegerField() + success_count = serializers.IntegerField() + success_rate = serializers.FloatField() diff --git a/bkflow/statistics/signals/__init__.py b/bkflow/statistics/signals/__init__.py new file mode 100644 index 0000000000..5906eb478f --- /dev/null +++ b/bkflow/statistics/signals/__init__.py @@ -0,0 +1,22 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from bkflow.statistics.signals.handlers import register_statistics_signals + +__all__ = ["register_statistics_signals"] diff --git a/bkflow/statistics/signals/handlers.py b/bkflow/statistics/signals/handlers.py new file mode 100644 index 0000000000..e6df8b9104 --- /dev/null +++ b/bkflow/statistics/signals/handlers.py @@ -0,0 +1,116 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging + +from django.db.models.signals import post_save +from django.dispatch import receiver + +from bkflow.statistics.conf import StatisticsConfig + +logger = logging.getLogger("celery") + +# 标记信号是否已注册 +_signals_registered = False + + +def register_statistics_signals(): + """ + 根据模块类型注册统计信号 + 在 apps.py 的 ready() 方法中调用 + """ + global _signals_registered + if _signals_registered: + return + + if not StatisticsConfig.is_enabled(): + logger.info("[Statistics] Statistics is disabled, skip signal registration") + return + + if StatisticsConfig.should_collect_template_stats(): + _register_template_signals() + logger.info("[Statistics] Template statistics signals registered") + + if StatisticsConfig.should_collect_task_stats(): + _register_task_signals() + logger.info("[Statistics] Task statistics signals registered") + + _signals_registered = True + + +def _register_template_signals(): + """注册模板统计信号（Interface 模块）""" + try: + from bkflow.template.models import Template + + @receiver(post_save, sender=Template, dispatch_uid="template_statistics_post_save") + def template_post_save_handler(sender, instance, created, **kwargs): + """模板保存后触发统计""" + try: + from bkflow.statistics.tasks import template_post_save_statistics_task + + template_post_save_statistics_task.delay(template_id=instance.id) + except Exception as e: + logger.exception(f"[template_post_save_handler] template_id={instance.id} error: {e}") + + except ImportError as e: + logger.warning(f"[Statistics] Cannot register template signals: {e}") + + +def _register_task_signals(): + """注册任务统计信号（Engine 模块）""" + try: + from bkflow.task.models import TaskInstance + + @receiver(post_save, sender=TaskInstance, dispatch_uid="task_statistics_post_save") + def task_post_save_handler(sender, instance, created, **kwargs): + """任务创建后触发统计""" + if created: + try: + from bkflow.statistics.tasks import task_created_statistics_task + + task_created_statistics_task.delay(task_id=instance.id) + except Exception as e: + logger.exception(f"[task_post_save_handler] task_id={instance.id} error: {e}") + + # 注册 pipeline 状态变更信号 + try: + from bamboo_engine import states as bamboo_engine_states + from pipeline.eri.signals import post_set_state + + @receiver(post_set_state, dispatch_uid="task_statistics_state_change") + def task_state_change_handler(sender, node_id, to_state, version, root_id, **kwargs): + """任务状态变更时触发统计""" + # 只在任务完成或撤销时采集执行统计 + if node_id == root_id and to_state in ( + bamboo_engine_states.FINISHED, + bamboo_engine_states.REVOKED, + ): + try: + from bkflow.statistics.tasks import task_archive_statistics_task + + task_archive_statistics_task.delay(instance_id=root_id) + except Exception as e: + logger.exception(f"[task_state_change_handler] instance_id={root_id} error: {e}") + + except ImportError as e: + logger.warning(f"[Statistics] Cannot register pipeline state signals: {e}") + + except ImportError as e: + logger.warning(f"[Statistics] Cannot register task signals: {e}") diff --git a/bkflow/statistics/tasks/__init__.py b/bkflow/statistics/tasks/__init__.py new file mode 100644 index 0000000000..8714baba14 --- /dev/null +++ b/bkflow/statistics/tasks/__init__.py @@ -0,0 +1,38 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from bkflow.statistics.tasks.summary_tasks import ( + clean_expired_statistics_task, + generate_daily_summary_task, + generate_plugin_summary_task, +) +from bkflow.statistics.tasks.task_tasks import ( + task_archive_statistics_task, + task_created_statistics_task, +) +from bkflow.statistics.tasks.template_tasks import template_post_save_statistics_task + +__all__ = [ + "template_post_save_statistics_task", + "task_created_statistics_task", + "task_archive_statistics_task", + "generate_daily_summary_task", + "generate_plugin_summary_task", + "clean_expired_statistics_task", +] diff --git a/bkflow/statistics/tasks/summary_tasks.py b/bkflow/statistics/tasks/summary_tasks.py new file mode 100644 index 0000000000..5db4dce977 --- /dev/null +++ b/bkflow/statistics/tasks/summary_tasks.py @@ -0,0 +1,227 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging +from datetime import date, timedelta + +from celery import shared_task +from django.db.models import Avg, Count, Max, Q, Sum + +from bkflow.statistics.conf import StatisticsConfig +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, +) + +logger = logging.getLogger("celery") + + +@shared_task(bind=True, ignore_result=True) +def generate_daily_summary_task(self, target_date: str = None): + """ + 生成每日统计汇总 + + Args: + target_date: 目标日期（YYYY-MM-DD），默认为昨天 + """ + if not StatisticsConfig.is_enabled(): + logger.debug("[daily_summary] Statistics is disabled") + return + + if target_date: + summary_date = date.fromisoformat(target_date) + else: + summary_date = date.today() - timedelta(days=1) + + try: + _generate_daily_summary(summary_date) + logger.info(f"[daily_summary] date={summary_date} generated successfully") + except Exception as e: + logger.exception(f"[daily_summary] date={summary_date} error: {e}") + + +def _generate_daily_summary(summary_date: date): + """生成指定日期的汇总数据""" + db_alias = StatisticsConfig.get_db_alias() + + # 按 space_id + scope 分组统计 + task_stats = ( + TaskflowStatistics.objects.using(db_alias) + .filter(create_time__date=summary_date) + .values("space_id", "scope_type", "scope_value") + .annotate( + task_created=Count("id"), + task_started=Count("id", filter=Q(is_started=True)), + task_finished=Count("id", filter=Q(is_finished=True)), + task_success=Count("id", filter=Q(is_success=True)), + task_failed=Count("id", filter=Q(is_finished=True, is_success=False)), + task_revoked=Count("id", filter=Q(final_state="REVOKED")), + avg_elapsed=Avg("elapsed_time", filter=Q(elapsed_time__isnull=False)), + max_elapsed=Max("elapsed_time"), + total_elapsed=Sum("elapsed_time", filter=Q(elapsed_time__isnull=False)), + ) + ) + + for stat in task_stats: + # 获取节点统计 + node_stats = ( + TaskflowExecutedNodeStatistics.objects.using(db_alias) + .filter( + space_id=stat["space_id"], + scope_type=stat["scope_type"], + scope_value=stat["scope_value"], + started_time__date=summary_date, + ) + .aggregate( + node_executed=Count("id", filter=Q(is_retry=False)), + node_success=Count("id", filter=Q(status=True, is_retry=False)), + node_failed=Count("id", filter=Q(status=False, is_retry=False)), + node_retry=Count("id", filter=Q(is_retry=True)), + ) + ) + + DailyStatisticsSummary.objects.using(db_alias).update_or_create( + date=summary_date, + space_id=stat["space_id"], + scope_type=stat["scope_type"] or "", + scope_value=stat["scope_value"] or "", + defaults={ + "task_created_count": stat["task_created"], + "task_started_count": stat["task_started"], + "task_finished_count": stat["task_finished"], + "task_success_count": stat["task_success"], + "task_failed_count": stat["task_failed"], + "task_revoked_count": stat["task_revoked"], + "avg_task_elapsed_time": stat["avg_elapsed"] or 0, + "max_task_elapsed_time": stat["max_elapsed"] or 0, + "total_task_elapsed_time": stat["total_elapsed"] or 0, + "node_executed_count": node_stats["node_executed"] or 0, + "node_success_count": node_stats["node_success"] or 0, + "node_failed_count": node_stats["node_failed"] or 0, + "node_retry_count": node_stats["node_retry"] or 0, + }, + ) + + +@shared_task(bind=True, ignore_result=True) +def generate_plugin_summary_task(self, period_type: str = "day", target_date: str = None): + """ + 生成插件执行汇总 + + Args: + period_type: 周期类型（day/week/month） + target_date: 周期开始日期 + """ + if not StatisticsConfig.is_enabled(): + logger.debug("[plugin_summary] Statistics is disabled") + return + + if target_date: + period_start = date.fromisoformat(target_date) + else: + period_start = date.today() - timedelta(days=1) + + try: + _generate_plugin_summary(period_type, period_start) + logger.info(f"[plugin_summary] {period_type}={period_start} generated successfully") + except Exception as e: + logger.exception(f"[plugin_summary] {period_type}={period_start} error: {e}") + + +def _generate_plugin_summary(period_type: str, period_start: date): + """生成插件执行汇总""" + db_alias = StatisticsConfig.get_db_alias() + + # 确定时间范围 + if period_type == "day": + period_end = period_start + timedelta(days=1) + elif period_type == "week": + period_end = period_start + timedelta(weeks=1) + else: # month + period_end = period_start + timedelta(days=30) + + # 按 space_id + component_code + version 分组统计 + node_stats = ( + TaskflowExecutedNodeStatistics.objects.using(db_alias) + .filter(started_time__date__gte=period_start, started_time__date__lt=period_end) + .values("space_id", "component_code", "version", "is_remote") + .annotate( + execution=Count("id"), + success=Count("id", filter=Q(status=True)), + failed=Count("id", filter=Q(status=False)), + retry=Count("id", filter=Q(is_retry=True)), + timeout=Count("id", filter=Q(is_timeout=True)), + avg_elapsed=Avg("elapsed_time", filter=Q(elapsed_time__isnull=False)), + max_elapsed=Max("elapsed_time"), + ) + ) + + for stat in node_stats: + PluginExecutionSummary.objects.using(db_alias).update_or_create( + period_type=period_type, + period_start=period_start, + space_id=stat["space_id"], + component_code=stat["component_code"], + version=stat["version"], + defaults={ + "is_remote": stat["is_remote"], + "execution_count": stat["execution"], + "success_count": stat["success"], + "failed_count": stat["failed"], + "retry_count": stat["retry"], + "timeout_count": stat["timeout"], + "avg_elapsed_time": stat["avg_elapsed"] or 0, + "max_elapsed_time": stat["max_elapsed"] or 0, + }, + ) + + +@shared_task(bind=True, ignore_result=True) +def clean_expired_statistics_task(self): + """清理过期统计数据""" + if not StatisticsConfig.is_enabled(): + return + + retention_days = StatisticsConfig.get_retention_days() + if retention_days <= 0: + logger.info("[clean_statistics] Retention days is 0, skip cleanup") + return + + db_alias = StatisticsConfig.get_db_alias() + cutoff_date = date.today() - timedelta(days=retention_days) + + try: + # 清理节点执行统计 + deleted_nodes, _ = ( + TaskflowExecutedNodeStatistics.objects.using(db_alias).filter(started_time__date__lt=cutoff_date).delete() + ) + + # 清理每日汇总 + deleted_daily, _ = DailyStatisticsSummary.objects.using(db_alias).filter(date__lt=cutoff_date).delete() + + # 清理插件汇总 + deleted_plugin, _ = PluginExecutionSummary.objects.using(db_alias).filter(period_start__lt=cutoff_date).delete() + + logger.info( + f"[clean_statistics] Cleaned: nodes={deleted_nodes}, daily={deleted_daily}, plugin={deleted_plugin}" + ) + except Exception as e: + logger.exception(f"[clean_statistics] error: {e}") diff --git a/bkflow/statistics/tasks/task_tasks.py b/bkflow/statistics/tasks/task_tasks.py new file mode 100644 index 0000000000..61bba18ef7 --- /dev/null +++ b/bkflow/statistics/tasks/task_tasks.py @@ -0,0 +1,77 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging + +from celery import shared_task + +from bkflow.statistics.conf import StatisticsConfig + +logger = logging.getLogger("celery") + + +@shared_task(bind=True, ignore_result=True) +def task_created_statistics_task(self, task_id: int): + """ + 任务创建后的统计任务 + + 采集内容： + - TaskflowStatistics: 任务基本信息 + """ + if not StatisticsConfig.is_enabled(): + logger.debug("[task_statistics] Statistics is disabled") + return + + try: + from bkflow.statistics.collectors import TaskStatisticsCollector + + collector = TaskStatisticsCollector(task_id=task_id) + result = collector.collect_on_create() + if result: + logger.info(f"[task_statistics] task_id={task_id} created stats collected") + else: + logger.debug(f"[task_statistics] task_id={task_id} skipped or failed") + except Exception as e: + logger.exception(f"[task_statistics] task_id={task_id} error: {e}") + + +@shared_task(bind=True, ignore_result=True) +def task_archive_statistics_task(self, instance_id: str): + """ + 任务归档统计任务（完成/撤销后触发） + + 采集内容： + - TaskflowStatistics: 更新执行信息 + - TaskflowExecutedNodeStatistics: 节点执行详情 + """ + if not StatisticsConfig.is_enabled(): + logger.debug("[task_statistics] Statistics is disabled") + return + + try: + from bkflow.statistics.collectors import TaskStatisticsCollector + + collector = TaskStatisticsCollector(instance_id=instance_id) + result = collector.collect_on_archive() + if result: + logger.info(f"[task_statistics] instance_id={instance_id} archive stats collected") + else: + logger.warning(f"[task_statistics] instance_id={instance_id} archive collection failed") + except Exception as e: + logger.exception(f"[task_statistics] instance_id={instance_id} error: {e}") diff --git a/bkflow/statistics/tasks/template_tasks.py b/bkflow/statistics/tasks/template_tasks.py new file mode 100644 index 0000000000..c9e6d94853 --- /dev/null +++ b/bkflow/statistics/tasks/template_tasks.py @@ -0,0 +1,52 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +import logging + +from celery import shared_task + +from bkflow.statistics.conf import StatisticsConfig + +logger = logging.getLogger("celery") + + +@shared_task(bind=True, ignore_result=True) +def template_post_save_statistics_task(self, template_id: int): + """ + 模板保存后的统计任务 + + 采集内容： + - TemplateNodeStatistics: 模板节点引用统计 + - TemplateStatistics: 模板整体统计 + """ + if not StatisticsConfig.is_enabled(): + logger.debug("[template_statistics] Statistics is disabled") + return + + try: + from bkflow.statistics.collectors import TemplateStatisticsCollector + + collector = TemplateStatisticsCollector(template_id=template_id) + result = collector.collect() + if result: + logger.info(f"[template_statistics] template_id={template_id} collected successfully") + else: + logger.warning(f"[template_statistics] template_id={template_id} collection failed") + except Exception as e: + logger.exception(f"[template_statistics] template_id={template_id} error: {e}") diff --git a/bkflow/statistics/urls.py b/bkflow/statistics/urls.py new file mode 100644 index 0000000000..33be26be75 --- /dev/null +++ b/bkflow/statistics/urls.py @@ -0,0 +1,30 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from bkflow.statistics.views import SpaceStatisticsViewSet + +router = DefaultRouter() +router.register(r"space", SpaceStatisticsViewSet, basename="space-statistics") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/bkflow/statistics/views.py b/bkflow/statistics/views.py new file mode 100644 index 0000000000..0aac064823 --- /dev/null +++ b/bkflow/statistics/views.py @@ -0,0 +1,375 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import date, timedelta + +from django.db.models import Avg, Count, Q, Sum +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.viewsets import ViewSet + +from bkflow.statistics.conf import StatisticsConfig +from bkflow.statistics.models import ( + DailyStatisticsSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, + TemplateStatistics, +) +from bkflow.statistics.serializers import ( + DailyStatisticsSummarySerializer, + PluginRankingSerializer, + StatisticsOverviewSerializer, + TaskTrendSerializer, + TemplateRankingSerializer, +) + + +class SpaceStatisticsViewSet(ViewSet): + """空间运营数据 API""" + + def get_db_alias(self): + return StatisticsConfig.get_db_alias() + + def _get_date_range(self, request): + """获取查询的日期范围""" + date_range = request.query_params.get("date_range", "30d") + end_date = date.today() + + if date_range == "7d": + start_date = end_date - timedelta(days=7) + elif date_range == "30d": + start_date = end_date - timedelta(days=30) + elif date_range == "90d": + start_date = end_date - timedelta(days=90) + else: + start_date = end_date - timedelta(days=30) + + return start_date, end_date + + @action(methods=["GET"], detail=False, url_path="overview") + def overview(self, request): + """ + 空间运营概览 + + Query params: + space_id: 空间ID（必填） + scope_type: 范围类型（可选） + scope_value: 范围值（可选） + date_range: 日期范围（7d/30d/90d，默认30d） + """ + space_id = request.query_params.get("space_id") + if not space_id: + return Response({"error": "space_id is required"}, status=status.HTTP_400_BAD_REQUEST) + + scope_type = request.query_params.get("scope_type") + scope_value = request.query_params.get("scope_value") + start_date, end_date = self._get_date_range(request) + + db_alias = self.get_db_alias() + + # 构建查询条件 + task_filters = Q(space_id=space_id, create_time__date__gte=start_date, create_time__date__lte=end_date) + if scope_type: + task_filters &= Q(scope_type=scope_type) + if scope_value: + task_filters &= Q(scope_value=scope_value) + + # 任务统计 + task_stats = ( + TaskflowStatistics.objects.using(db_alias) + .filter(task_filters) + .aggregate( + total_tasks=Count("id"), + success_tasks=Count("id", filter=Q(is_success=True)), + failed_tasks=Count("id", filter=Q(is_finished=True, is_success=False)), + avg_elapsed=Avg("elapsed_time", filter=Q(elapsed_time__isnull=False)), + ) + ) + + # 模板统计 + template_filters = Q(space_id=space_id) + if scope_type: + template_filters &= Q(scope_type=scope_type) + if scope_value: + template_filters &= Q(scope_value=scope_value) + + template_stats = ( + TemplateStatistics.objects.using(db_alias) + .filter(template_filters) + .aggregate( + total_templates=Count("id"), + active_templates=Count("id", filter=Q(is_enabled=True)), + ) + ) + + # 节点统计 + node_filters = Q(space_id=space_id, started_time__date__gte=start_date, started_time__date__lte=end_date) + if scope_type: + node_filters &= Q(scope_type=scope_type) + if scope_value: + node_filters &= Q(scope_value=scope_value) + + node_stats = ( + TaskflowExecutedNodeStatistics.objects.using(db_alias) + .filter(node_filters, is_retry=False) + .aggregate( + total_nodes=Count("id"), + success_nodes=Count("id", filter=Q(status=True)), + ) + ) + + total_tasks = task_stats["total_tasks"] or 0 + success_tasks = task_stats["success_tasks"] or 0 + total_nodes = node_stats["total_nodes"] or 0 + success_nodes = node_stats["success_nodes"] or 0 + + result = { + "space_id": int(space_id), + "scope_type": scope_type, + "scope_value": scope_value, + "total_tasks": total_tasks, + "success_tasks": success_tasks, + "failed_tasks": task_stats["failed_tasks"] or 0, + "success_rate": round(success_tasks / total_tasks * 100, 2) if total_tasks > 0 else 0, + "total_templates": template_stats["total_templates"] or 0, + "active_templates": template_stats["active_templates"] or 0, + "total_nodes_executed": total_nodes, + "node_success_rate": round(success_nodes / total_nodes * 100, 2) if total_nodes > 0 else 0, + "avg_task_elapsed_time": round(task_stats["avg_elapsed"] or 0, 2), + } + + serializer = StatisticsOverviewSerializer(result) + return Response(serializer.data) + + @action(methods=["GET"], detail=False, url_path="task-trend") + def task_trend(self, request): + """ + 任务执行趋势 + + Query params: + space_id: 空间ID（必填） + scope_type: 范围类型（可选） + scope_value: 范围值（可选） + date_range: 日期范围（7d/30d/90d，默认30d） + """ + space_id = request.query_params.get("space_id") + if not space_id: + return Response({"error": "space_id is required"}, status=status.HTTP_400_BAD_REQUEST) + + scope_type = request.query_params.get("scope_type") + scope_value = request.query_params.get("scope_value") + start_date, end_date = self._get_date_range(request) + + db_alias = self.get_db_alias() + + # 优先从每日汇总表查询 + filters = Q(space_id=space_id, date__gte=start_date, date__lte=end_date) + if scope_type: + filters &= Q(scope_type=scope_type) + if scope_value: + filters &= Q(scope_value=scope_value) + + daily_stats = ( + DailyStatisticsSummary.objects.using(db_alias) + .filter(filters) + .values("date") + .annotate( + task_created_count=Sum("task_created_count"), + task_finished_count=Sum("task_finished_count"), + task_success_count=Sum("task_success_count"), + ) + .order_by("date") + ) + + result = [] + for stat in daily_stats: + finished = stat["task_finished_count"] or 0 + success = stat["task_success_count"] or 0 + result.append( + { + "date": stat["date"], + "task_created_count": stat["task_created_count"] or 0, + "task_finished_count": finished, + "task_success_count": success, + "success_rate": round(success / finished * 100, 2) if finished > 0 else 0, + } + ) + + serializer = TaskTrendSerializer(result, many=True) + return Response(serializer.data) + + @action(methods=["GET"], detail=False, url_path="plugin-ranking") + def plugin_ranking(self, request): + """ + 插件排行 + + Query params: + space_id: 空间ID（必填） + date_range: 日期范围（7d/30d/90d，默认30d） + order_by: 排序字段（execution_count/success_rate/avg_elapsed_time，默认execution_count） + limit: 返回数量（默认10） + """ + space_id = request.query_params.get("space_id") + if not space_id: + return Response({"error": "space_id is required"}, status=status.HTTP_400_BAD_REQUEST) + + start_date, end_date = self._get_date_range(request) + order_by = request.query_params.get("order_by", "execution_count") + limit = int(request.query_params.get("limit", 10)) + + db_alias = self.get_db_alias() + + filters = Q(space_id=space_id, started_time__date__gte=start_date, started_time__date__lte=end_date) + + plugin_stats = ( + TaskflowExecutedNodeStatistics.objects.using(db_alias) + .filter(filters, is_retry=False) + .values("component_code", "version") + .annotate( + execution_count=Count("id"), + success_count=Count("id", filter=Q(status=True)), + failed_count=Count("id", filter=Q(status=False)), + avg_elapsed_time=Avg("elapsed_time", filter=Q(elapsed_time__isnull=False)), + ) + .order_by(f"-{order_by}" if order_by != "success_rate" else "-execution_count")[:limit] + ) + + result = [] + for stat in plugin_stats: + execution = stat["execution_count"] or 0 + success = stat["success_count"] or 0 + result.append( + { + "component_code": stat["component_code"], + "version": stat["version"], + "execution_count": execution, + "success_count": success, + "failed_count": stat["failed_count"] or 0, + "success_rate": round(success / execution * 100, 2) if execution > 0 else 0, + "avg_elapsed_time": round(stat["avg_elapsed_time"] or 0, 2), + } + ) + + # 如果按成功率排序，需要在内存中重新排序 + if order_by == "success_rate": + result.sort(key=lambda x: x["success_rate"], reverse=True) + + serializer = PluginRankingSerializer(result, many=True) + return Response(serializer.data) + + @action(methods=["GET"], detail=False, url_path="template-ranking") + def template_ranking(self, request): + """ + 模板排行 + + Query params: + space_id: 空间ID（必填） + date_range: 日期范围（7d/30d/90d，默认30d） + order_by: 排序字段（task_count/success_rate，默认task_count） + limit: 返回数量（默认10） + """ + space_id = request.query_params.get("space_id") + if not space_id: + return Response({"error": "space_id is required"}, status=status.HTTP_400_BAD_REQUEST) + + start_date, end_date = self._get_date_range(request) + order_by = request.query_params.get("order_by", "task_count") + limit = int(request.query_params.get("limit", 10)) + + db_alias = self.get_db_alias() + + filters = Q( + space_id=space_id, + template_id__isnull=False, + create_time__date__gte=start_date, + create_time__date__lte=end_date, + ) + + template_stats = ( + TaskflowStatistics.objects.using(db_alias) + .filter(filters) + .values("template_id") + .annotate( + task_count=Count("id"), + success_count=Count("id", filter=Q(is_success=True)), + ) + .order_by("-task_count")[:limit] + ) + + # 获取模板名称 + template_ids = [stat["template_id"] for stat in template_stats] + template_names = dict( + TemplateStatistics.objects.using(db_alias) + .filter(template_id__in=template_ids) + .values_list("template_id", "template_name") + ) + + result = [] + for stat in template_stats: + task_count = stat["task_count"] or 0 + success_count = stat["success_count"] or 0 + result.append( + { + "template_id": stat["template_id"], + "template_name": template_names.get(stat["template_id"], ""), + "task_count": task_count, + "success_count": success_count, + "success_rate": round(success_count / task_count * 100, 2) if task_count > 0 else 0, + } + ) + + # 如果按成功率排序，需要在内存中重新排序 + if order_by == "success_rate": + result.sort(key=lambda x: x["success_rate"], reverse=True) + + serializer = TemplateRankingSerializer(result, many=True) + return Response(serializer.data) + + @action(methods=["GET"], detail=False, url_path="daily-summary") + def daily_summary(self, request): + """ + 每日统计汇总 + + Query params: + space_id: 空间ID（必填） + scope_type: 范围类型（可选） + scope_value: 范围值（可选） + date_range: 日期范围（7d/30d/90d，默认30d） + """ + space_id = request.query_params.get("space_id") + if not space_id: + return Response({"error": "space_id is required"}, status=status.HTTP_400_BAD_REQUEST) + + scope_type = request.query_params.get("scope_type") + scope_value = request.query_params.get("scope_value") + start_date, end_date = self._get_date_range(request) + + db_alias = self.get_db_alias() + + filters = Q(space_id=space_id, date__gte=start_date, date__lte=end_date) + if scope_type: + filters &= Q(scope_type=scope_type) + if scope_value: + filters &= Q(scope_value=scope_value) + + daily_stats = DailyStatisticsSummary.objects.using(db_alias).filter(filters).order_by("date") + + serializer = DailyStatisticsSummarySerializer(daily_stats, many=True) + return Response(serializer.data) diff --git a/bkflow/urls.py b/bkflow/urls.py index b8c5240f50..8d464d3a5e 100644 --- a/bkflow/urls.py +++ b/bkflow/urls.py @@ -44,6 +44,17 @@ url(r"^notice/", include("bk_notice_sdk.urls")), url(r"^version_log/", include("version_log.urls", namespace="version_log")), ] + + # 统计 API 路由 + try: + import env + + if getattr(env, "STATISTICS_ENABLED", True): + urlpatterns += [ + url(r"^api/statistics/", include("bkflow.statistics.urls")), + ] + except ImportError: + pass elif settings.BKFLOW_MODULE.type == BKFLOWModuleType.engine: engine_admin_actions = [ "task_pause", diff --git a/env.py b/env.py index fde9f5dea1..c24cdfdd60 100644 --- a/env.py +++ b/env.py @@ -181,3 +181,29 @@ TEMPLATE_MAX_RECURSIVE_NUMBER = int(os.getenv("TEMPLATE_MAX_RECURSIVE_NUMBER", 10)) REQUEST_RETRY_NUMBER = int(os.getenv("REQUEST_RETRY_NUMBER", 3)) + +# ============== 统计功能配置 ============== +# 是否启用统计功能 +STATISTICS_ENABLED = os.getenv("STATISTICS_ENABLED", "true").lower() == "true" + +# 是否使用独立统计数据库 +STATISTICS_USE_SEPARATE_DB = os.getenv("STATISTICS_USE_SEPARATE_DB", "false").lower() == "true" + +# 统计数据库配置 +STATISTICS_DB_HOST = os.getenv("STATISTICS_DB_HOST", "") +STATISTICS_DB_PORT = int(os.getenv("STATISTICS_DB_PORT", "3306")) if os.getenv("STATISTICS_DB_PORT") else 3306 +STATISTICS_DB_NAME = os.getenv("STATISTICS_DB_NAME", "bkflow_statistics") +STATISTICS_DB_USER = os.getenv("STATISTICS_DB_USER", "") +STATISTICS_DB_PASSWORD = os.getenv("STATISTICS_DB_PASSWORD", "") + +# 数据库连接池配置 +STATISTICS_DB_CONN_MAX_AGE = int(os.getenv("STATISTICS_DB_CONN_MAX_AGE", "300")) + +# 是否统计 Mock 任务 +STATISTICS_INCLUDE_MOCK = os.getenv("STATISTICS_INCLUDE_MOCK", "false").lower() == "true" + +# 是否统计已删除任务 +STATISTICS_INCLUDE_DELETED = os.getenv("STATISTICS_INCLUDE_DELETED", "false").lower() == "true" + +# 统计数据保留天数（用于清理，0 表示不清理） +STATISTICS_RETENTION_DAYS = int(os.getenv("STATISTICS_RETENTION_DAYS", "365")) diff --git a/module_settings.py b/module_settings.py index 1392cf067a..3d96b80cf4 100644 --- a/module_settings.py +++ b/module_settings.py @@ -157,6 +157,10 @@ def check_engine_admin_permission(request, *args, **kwargs): "bkflow.contrib.expired_cleaner", ) + # 统计模块 + if env.STATISTICS_ENABLED: + INSTALLED_APPS += ("bkflow.statistics",) + BKFLOW_CELERY_ROUTES = { "bkflow.contrib.expired_cleaner.tasks.clean_task": { "queue": f"clean_task_{BKFLOW_MODULE.code}", @@ -245,6 +249,10 @@ def check_engine_admin_permission(request, *args, **kwargs): "bkflow.pipeline_web", ) + # 统计模块 + if env.STATISTICS_ENABLED: + INSTALLED_APPS += ("bkflow.statistics",) + VARIABLE_KEY_BLACKLIST = ( env.VARIABLE_KEY_BLACKLIST.strip().strip(",").split(",") if env.VARIABLE_KEY_BLACKLIST else [] ) @@ -288,3 +296,45 @@ def check_engine_admin_permission(request, *args, **kwargs): "schedule": crontab(env.SYNC_BK_PLUGINS_CRONTAB), } } + + # 如果启用统计功能，添加统计汇总定时任务 + if env.STATISTICS_ENABLED: + app.conf.beat_schedule.update( + { + "generate_daily_summary": { + "task": "bkflow.statistics.tasks.generate_daily_summary_task", + "schedule": crontab(minute=30, hour=1), # 每天凌晨 1:30 执行 + }, + "generate_plugin_summary": { + "task": "bkflow.statistics.tasks.generate_plugin_summary_task", + "schedule": crontab(minute=0, hour=2), # 每天凌晨 2:00 执行 + }, + "clean_expired_statistics": { + "task": "bkflow.statistics.tasks.clean_expired_statistics_task", + "schedule": crontab(minute=0, hour=3), # 每天凌晨 3:00 执行 + }, + } + ) + + +# 统计数据库配置（适用于所有模块类型） +if env.STATISTICS_ENABLED and env.STATISTICS_USE_SEPARATE_DB and env.STATISTICS_DB_HOST: + # 获取或初始化 DATABASES + if "DATABASES" not in dir(): + from config.default import DATABASES + + DATABASES["statistics"] = { + "ENGINE": "django.db.backends.mysql", + "NAME": env.STATISTICS_DB_NAME, + "USER": env.STATISTICS_DB_USER, + "PASSWORD": env.STATISTICS_DB_PASSWORD, + "HOST": env.STATISTICS_DB_HOST, + "PORT": env.STATISTICS_DB_PORT, + "CONN_MAX_AGE": env.STATISTICS_DB_CONN_MAX_AGE, + "OPTIONS": { + "charset": "utf8mb4", + }, + } + + # 添加数据库路由 + DATABASE_ROUTERS = ["bkflow.statistics.db_router.StatisticsDBRouter"] diff --git a/scripts/run_all_unit_test.sh b/scripts/run_all_unit_test.sh index 0102243849..57f5f5a4b5 100644 --- a/scripts/run_all_unit_test.sh +++ b/scripts/run_all_unit_test.sh @@ -1,6 +1,4 @@ #!/bin/bash -source /root/.envs/bkflow/bin/activate - set -e export $(cat tests/interface.env | xargs) echo "开始运行${BKFLOW_MODULE_TYPE}测试" @@ -11,4 +9,4 @@ set -e export $(cat tests/engine.env | xargs) echo "开始运行${BKFLOW_MODULE_TYPE}测试" pytest --cov-append tests/engine tests/plugin_service -set +e \ No newline at end of file +set +e diff --git a/scripts/run_engine_unit_test.sh b/scripts/run_engine_unit_test.sh index 51eb840434..a796f9154b 100644 --- a/scripts/run_engine_unit_test.sh +++ b/scripts/run_engine_unit_test.sh @@ -1,5 +1,5 @@ #!/bin/bash set -e export $(cat tests/engine.env | xargs) -echo $BKFLOW_MODULE_TYPE +echo "Running ${BKFLOW_MODULE_TYPE} unit tests..." pytest --cov-append tests/engine tests/plugin_service \ No newline at end of file diff --git a/scripts/run_interface_unit_test.sh b/scripts/run_interface_unit_test.sh index 08094496d6..4d6fadf24e 100644 --- a/scripts/run_interface_unit_test.sh +++ b/scripts/run_interface_unit_test.sh @@ -1,5 +1,5 @@ #!/bin/bash set -e export $(cat tests/interface.env | xargs) -echo $BKFLOW_MODULE_TYPE +echo "Running ${BKFLOW_MODULE_TYPE} unit tests..." pytest tests/interface tests/plugins tests/project_settings tests/contrib tests/decision_table diff --git a/tests/engine/statistics/__init__.py b/tests/engine/statistics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/engine/statistics/test_collectors.py b/tests/engine/statistics/test_collectors.py new file mode 100644 index 0000000000..83e85a2f5e --- /dev/null +++ b/tests/engine/statistics/test_collectors.py @@ -0,0 +1,124 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import MagicMock, patch + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.collectors import TaskStatisticsCollector +from bkflow.statistics.collectors.base import BaseStatisticsCollector +from bkflow.statistics.models import TaskflowStatistics + + +class TestTaskStatisticsCollector(TestCase): + """任务统计采集器测试""" + + def test_collector_initialization(self): + """测试采集器初始化""" + collector = TaskStatisticsCollector(task_id=1) + self.assertEqual(collector.task_id, 1) + self.assertIsNone(collector.instance_id) + + collector2 = TaskStatisticsCollector(instance_id="instance_001") + self.assertIsNone(collector2.task_id) + self.assertEqual(collector2.instance_id, "instance_001") + + def test_collect_without_task(self): + """测试任务不存在时的采集""" + collector = TaskStatisticsCollector(task_id=999) + collector._task = None + result = collector.collect_on_create() + self.assertFalse(result) + + @patch("bkflow.statistics.conf.StatisticsConfig.include_mock_tasks") + def test_skip_mock_task(self, mock_include_mock): + """测试跳过 Mock 任务""" + mock_include_mock.return_value = False + + collector = TaskStatisticsCollector(task_id=1) + + # 创建 Mock 任务 + mock_task = MagicMock() + mock_task.id = 1 + mock_task.create_method = "MOCK" + + collector._task = mock_task + result = collector.collect_on_create() + self.assertFalse(result) + + @patch("bkflow.statistics.conf.StatisticsConfig.include_mock_tasks") + @patch("bkflow.statistics.conf.StatisticsConfig.get_engine_id") + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_collect_on_create_success(self, mock_db_alias, mock_engine_id, mock_include_mock): + """测试成功创建任务统计""" + mock_include_mock.return_value = True + mock_engine_id.return_value = "test_engine" + mock_db_alias.return_value = "default" + + collector = TaskStatisticsCollector(task_id=1) + + # 创建 mock 任务 + mock_task = MagicMock() + mock_task.id = 1 + mock_task.instance_id = "instance_001" + mock_task.template_id = 1 + mock_task.space_id = 1 + mock_task.scope_type = "biz" + mock_task.scope_value = "2" + mock_task.creator = "admin" + mock_task.executor = "admin" + mock_task.create_time = timezone.now() + mock_task.create_method = "API" + mock_task.trigger_method = "manual" + mock_task.is_started = False + mock_task.execution_data = { + "activities": { + "node_1": {"type": "ServiceActivity"}, + }, + "gateways": {}, + } + + collector._task = mock_task + + result = collector.collect_on_create() + self.assertTrue(result) + + # 验证是否创建了统计记录 + stat = TaskflowStatistics.objects.get(task_id=1) + self.assertEqual(stat.instance_id, "instance_001") + self.assertEqual(stat.engine_id, "test_engine") + + def test_count_nodes_in_pipeline_tree(self): + """测试统计 pipeline_tree 中的节点""" + pipeline_tree = { + "activities": { + "node_1": {"type": "ServiceActivity"}, + "node_2": {"type": "ServiceActivity"}, + "node_3": {"type": "SubProcess", "pipeline": {}}, + }, + "gateways": { + "gateway_1": {"type": "ParallelGateway"}, + }, + } + + atom, subprocess, gateways = BaseStatisticsCollector.count_pipeline_tree_nodes(pipeline_tree) + self.assertEqual(atom, 2) + self.assertEqual(subprocess, 1) + self.assertEqual(gateways, 1) diff --git a/tests/engine/statistics/test_conf.py b/tests/engine/statistics/test_conf.py new file mode 100644 index 0000000000..727964cae1 --- /dev/null +++ b/tests/engine/statistics/test_conf.py @@ -0,0 +1,73 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import patch + +from django.test import TestCase, override_settings + +from bkflow.statistics.conf import StatisticsConfig + + +class TestStatisticsConfigForEngine(TestCase): + """Engine 模块统计配置测试""" + + @patch("env.STATISTICS_ENABLED", True) + @patch("env.BKFLOW_MODULE_TYPE", "engine") + def test_should_collect_task_stats_engine(self): + """测试 Engine 模块采集任务统计""" + result = StatisticsConfig.should_collect_task_stats() + self.assertTrue(result) + + @patch("env.STATISTICS_ENABLED", True) + @patch("env.BKFLOW_MODULE_TYPE", "interface") + def test_should_collect_task_stats_interface(self): + """测试 Interface 模块不采集任务统计""" + result = StatisticsConfig.should_collect_task_stats() + self.assertFalse(result) + + @patch("env.STATISTICS_ENABLED", False) + @patch("env.BKFLOW_MODULE_TYPE", "engine") + def test_should_collect_task_stats_disabled(self): + """测试统计功能禁用时不采集任务统计""" + result = StatisticsConfig.should_collect_task_stats() + self.assertFalse(result) + + @patch("env.BKFLOW_MODULE_CODE", "engine_001") + def test_get_engine_id(self): + """测试获取 Engine ID""" + with patch.object(StatisticsConfig, "get_engine_id") as mock_method: + mock_method.return_value = "engine_001" + result = StatisticsConfig.get_engine_id() + self.assertEqual(result, "engine_001") + + def test_get_db_alias_default(self): + """测试获取默认数据库别名""" + result = StatisticsConfig.get_db_alias() + self.assertEqual(result, "default") + + @override_settings( + DATABASES={ + "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + "statistics": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + } + ) + def test_get_db_alias_with_statistics(self): + """测试配置了 statistics 数据库时的别名""" + result = StatisticsConfig.get_db_alias() + self.assertEqual(result, "statistics") diff --git a/tests/engine/statistics/test_models.py b/tests/engine/statistics/test_models.py new file mode 100644 index 0000000000..40744e3c62 --- /dev/null +++ b/tests/engine/statistics/test_models.py @@ -0,0 +1,183 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import timedelta + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.models import TaskflowExecutedNodeStatistics, TaskflowStatistics + + +class TestTaskflowStatistics(TestCase): + """任务统计模型测试""" + + def test_create_taskflow_statistics(self): + """测试创建任务统计""" + stat = TaskflowStatistics.objects.create( + task_id=1, + instance_id="instance_001", + template_id=1, + space_id=1, + engine_id="engine_a", + atom_total=5, + node_total=10, + creator="admin", + executor="admin", + create_time=timezone.now(), + create_method="API", + trigger_method="manual", + is_started=True, + is_finished=True, + is_success=True, + final_state="FINISHED", + ) + self.assertEqual(stat.task_id, 1) + self.assertEqual(stat.engine_id, "engine_a") + self.assertTrue(stat.is_success) + + def test_elapsed_time_calculation(self): + """测试耗时计算""" + start_time = timezone.now() + finish_time = start_time + timedelta(seconds=120) + + stat = TaskflowStatistics.objects.create( + task_id=2, + instance_id="instance_002", + space_id=1, + create_time=start_time, + start_time=start_time, + finish_time=finish_time, + elapsed_time=120, + ) + self.assertEqual(stat.elapsed_time, 120) + + def test_unique_task_id(self): + """测试任务ID唯一性""" + TaskflowStatistics.objects.create( + task_id=3, + instance_id="instance_003", + space_id=1, + create_time=timezone.now(), + ) + with self.assertRaises(Exception): + TaskflowStatistics.objects.create( + task_id=3, + instance_id="instance_003_dup", + space_id=1, + create_time=timezone.now(), + ) + + def test_str_representation(self): + """测试字符串表示""" + stat = TaskflowStatistics(task_id=1) + self.assertEqual(str(stat), "Task_1") + + +class TestTaskflowExecutedNodeStatistics(TestCase): + """节点执行统计模型测试""" + + def test_create_node_statistics(self): + """测试创建节点执行统计""" + stat = TaskflowExecutedNodeStatistics.objects.create( + component_code="test_component", + version="v1.0", + is_remote=False, + task_id=1, + instance_id="instance_001", + template_id=1, + space_id=1, + engine_id="engine_a", + node_id="node_1", + node_name="测试节点", + started_time=timezone.now(), + archived_time=timezone.now(), + elapsed_time=10, + status=True, + state="FINISHED", + is_skip=False, + retry_count=0, + task_create_time=timezone.now(), + ) + self.assertEqual(stat.component_code, "test_component") + self.assertTrue(stat.status) + self.assertEqual(stat.state, "FINISHED") + + def test_node_with_retry(self): + """测试重试节点""" + stat = TaskflowExecutedNodeStatistics.objects.create( + component_code="test_component", + version="v1.0", + task_id=1, + instance_id="instance_001", + space_id=1, + node_id="node_1", + started_time=timezone.now(), + status=False, + state="FAILED", + is_retry=True, + retry_count=2, + task_create_time=timezone.now(), + ) + self.assertTrue(stat.is_retry) + self.assertEqual(stat.retry_count, 2) + + def test_node_with_skip(self): + """测试跳过的节点""" + stat = TaskflowExecutedNodeStatistics.objects.create( + component_code="test_component", + version="v1.0", + task_id=1, + instance_id="instance_001", + space_id=1, + node_id="node_1", + started_time=timezone.now(), + status=True, + state="FINISHED", + is_skip=True, + task_create_time=timezone.now(), + ) + self.assertTrue(stat.is_skip) + + def test_remote_plugin_node(self): + """测试远程插件节点""" + stat = TaskflowExecutedNodeStatistics.objects.create( + component_code="remote_plugin_code", + version="v2.0", + is_remote=True, + task_id=1, + instance_id="instance_001", + space_id=1, + node_id="node_1", + started_time=timezone.now(), + status=True, + state="FINISHED", + task_create_time=timezone.now(), + ) + self.assertTrue(stat.is_remote) + self.assertEqual(stat.version, "v2.0") + + def test_str_representation(self): + """测试字符串表示""" + stat = TaskflowExecutedNodeStatistics( + component_code="test_component", + task_id=1, + node_id="node_1", + ) + self.assertEqual(str(stat), "test_component_1_node_1") diff --git a/tests/engine/statistics/test_tasks.py b/tests/engine/statistics/test_tasks.py new file mode 100644 index 0000000000..00fd0cca36 --- /dev/null +++ b/tests/engine/statistics/test_tasks.py @@ -0,0 +1,138 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import MagicMock, patch + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.collectors import TaskStatisticsCollector +from bkflow.statistics.models import TaskflowStatistics +from bkflow.statistics.tasks import ( + task_archive_statistics_task, + task_created_statistics_task, +) + + +class TestTaskStatisticsTasks(TestCase): + """任务统计任务测试（在 Engine 模块执行）""" + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + def test_task_created_task_disabled(self, mock_is_enabled): + """测试统计功能禁用时跳过采集""" + mock_is_enabled.return_value = False + task_created_statistics_task(task_id=1) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.collectors.TaskStatisticsCollector.collect_on_create") + def test_task_created_task_enabled(self, mock_collect, mock_is_enabled): + """测试任务创建统计""" + mock_is_enabled.return_value = True + mock_collect.return_value = True + task_created_statistics_task(task_id=1) + mock_collect.assert_called_once() + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + def test_task_archive_task_disabled(self, mock_is_enabled): + """测试统计功能禁用时跳过归档采集""" + mock_is_enabled.return_value = False + task_archive_statistics_task(instance_id="instance_001") + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.collectors.TaskStatisticsCollector.collect_on_archive") + def test_task_archive_task_enabled(self, mock_collect, mock_is_enabled): + """测试任务归档统计""" + mock_is_enabled.return_value = True + mock_collect.return_value = True + task_archive_statistics_task(instance_id="instance_001") + mock_collect.assert_called_once() + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.collectors.TaskStatisticsCollector.collect_on_create") + @patch("bkflow.statistics.collectors.TaskStatisticsCollector.collect_on_archive") + def test_task_full_lifecycle(self, mock_archive, mock_create, mock_is_enabled): + """测试任务完整生命周期统计""" + mock_is_enabled.return_value = True + mock_create.return_value = True + mock_archive.return_value = True + # 创建任务时采集 + task_created_statistics_task(task_id=1) + mock_create.assert_called_once() + # 归档任务时采集 + task_archive_statistics_task(instance_id="instance_001") + mock_archive.assert_called_once() + + +class TestTaskStatisticsCollectorIntegration(TestCase): + """任务统计采集器集成测试""" + + @patch("bkflow.statistics.conf.StatisticsConfig.get_engine_id") + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + @patch("bkflow.statistics.conf.StatisticsConfig.include_mock_tasks") + def test_collect_with_nested_subprocess(self, mock_include_mock, mock_db_alias, mock_engine_id): + """测试采集嵌套子流程的任务""" + mock_include_mock.return_value = True + mock_db_alias.return_value = "default" + mock_engine_id.return_value = "test_engine" + + collector = TaskStatisticsCollector(task_id=1) + + # 创建带嵌套子流程的 mock 任务 + mock_task = MagicMock() + mock_task.id = 1 + mock_task.instance_id = "instance_001" + mock_task.template_id = 1 + mock_task.space_id = 1 + mock_task.scope_type = "biz" + mock_task.scope_value = "2" + mock_task.creator = "admin" + mock_task.executor = "admin" + mock_task.create_time = timezone.now() + mock_task.create_method = "API" + mock_task.trigger_method = "manual" + mock_task.is_started = False + mock_task.execution_data = { + "activities": { + "node_1": {"type": "ServiceActivity"}, + "subprocess_1": { + "type": "SubProcess", + "pipeline": { + "activities": { + "sub_node_1": {"type": "ServiceActivity"}, + "sub_node_2": {"type": "ServiceActivity"}, + }, + "gateways": {}, + }, + }, + }, + "gateways": { + "gateway_1": {"type": "ParallelGateway"}, + }, + } + + collector._task = mock_task + + result = collector.collect_on_create() + self.assertTrue(result) + + stat = TaskflowStatistics.objects.get(task_id=1) + # 1 (root) + 2 (subprocess) = 3 atoms + self.assertEqual(stat.atom_total, 3) + self.assertEqual(stat.subprocess_total, 1) + self.assertEqual(stat.gateways_total, 1) diff --git a/tests/interface/statistics/__init__.py b/tests/interface/statistics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/interface/statistics/test_collectors.py b/tests/interface/statistics/test_collectors.py new file mode 100644 index 0000000000..5d7fcc0083 --- /dev/null +++ b/tests/interface/statistics/test_collectors.py @@ -0,0 +1,190 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import MagicMock + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.collectors import TemplateStatisticsCollector +from bkflow.statistics.collectors.base import BaseStatisticsCollector + + +class TestBaseStatisticsCollector(TestCase): + """基础统计采集器测试""" + + def test_count_pipeline_tree_nodes_empty(self): + """测试空 pipeline_tree 节点计数""" + pipeline_tree = {} + atom, subprocess, gateways = BaseStatisticsCollector.count_pipeline_tree_nodes(pipeline_tree) + self.assertEqual(atom, 0) + self.assertEqual(subprocess, 0) + self.assertEqual(gateways, 0) + + def test_count_pipeline_tree_nodes_with_activities(self): + """测试带有活动节点的 pipeline_tree 计数""" + pipeline_tree = { + "activities": { + "node_1": {"id": "node_1", "type": "ServiceActivity"}, + "node_2": {"id": "node_2", "type": "ServiceActivity"}, + "node_3": {"id": "node_3", "type": "SubProcess", "pipeline": {}}, + }, + "gateways": { + "gateway_1": {"id": "gateway_1", "type": "ParallelGateway"}, + "gateway_2": {"id": "gateway_2", "type": "ConvergeGateway"}, + }, + } + atom, subprocess, gateways = BaseStatisticsCollector.count_pipeline_tree_nodes(pipeline_tree) + self.assertEqual(atom, 2) + self.assertEqual(subprocess, 1) + self.assertEqual(gateways, 2) + + def test_count_pipeline_tree_nodes_with_nested_subprocess(self): + """测试嵌套子流程的节点计数""" + pipeline_tree = { + "activities": { + "node_1": {"id": "node_1", "type": "ServiceActivity"}, + "subprocess_1": { + "id": "subprocess_1", + "type": "SubProcess", + "pipeline": { + "activities": { + "sub_node_1": {"id": "sub_node_1", "type": "ServiceActivity"}, + "sub_node_2": {"id": "sub_node_2", "type": "ServiceActivity"}, + }, + "gateways": {}, + }, + }, + }, + "gateways": {}, + } + atom, subprocess, gateways = BaseStatisticsCollector.count_pipeline_tree_nodes(pipeline_tree) + self.assertEqual(atom, 3) # 1 + 2 from subprocess + self.assertEqual(subprocess, 1) + self.assertEqual(gateways, 0) + + def test_parse_datetime_with_valid_string(self): + """测试解析有效时间字符串""" + time_str = "2024-01-15T10:30:00+08:00" + result = BaseStatisticsCollector.parse_datetime(time_str) + self.assertIsNotNone(result) + + def test_parse_datetime_with_none(self): + """测试解析空值""" + result = BaseStatisticsCollector.parse_datetime(None) + self.assertIsNone(result) + + def test_parse_datetime_with_space_timezone(self): + """测试解析带空格时区的时间字符串""" + time_str = "2024-01-15 10:30:00 +0800" + result = BaseStatisticsCollector.parse_datetime(time_str) + # Should handle space-separated timezone + self.assertIsNotNone(result) + + +class TestTemplateStatisticsCollector(TestCase): + """模板统计采集器测试""" + + def test_collect_without_template(self): + """测试模板不存在时的采集""" + collector = TemplateStatisticsCollector(template_id=999) + collector._template = None # Force template to be None + result = collector.collect() + self.assertFalse(result) + + def test_collect_nodes_from_pipeline_tree(self): + """测试从 pipeline_tree 收集节点""" + collector = TemplateStatisticsCollector(template_id=1) + + # Create mock template + mock_template = MagicMock() + mock_template.id = 1 + mock_template.space_id = 1 + mock_template.scope_type = "biz" + mock_template.scope_value = "2" + mock_template.creator = "admin" + mock_template.create_at = timezone.now() + mock_template.update_at = timezone.now() + mock_template.pipeline_tree = { + "activities": { + "node_1": { + "id": "node_1", + "type": "ServiceActivity", + "name": "节点1", + "component": {"code": "test_component", "version": "v1.0"}, + }, + }, + "gateways": {}, + } + + collector._template = mock_template + + nodes = collector._collect_nodes( + pipeline_tree=mock_template.pipeline_tree, + subprocess_stack=[], + is_sub=False, + ) + + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].component_code, "test_component") + self.assertEqual(nodes[0].node_name, "节点1") + + def test_collect_remote_plugin_nodes(self): + """测试收集第三方插件节点""" + collector = TemplateStatisticsCollector(template_id=1) + + mock_template = MagicMock() + mock_template.id = 1 + mock_template.space_id = 1 + mock_template.scope_type = "biz" + mock_template.scope_value = "2" + mock_template.creator = "admin" + mock_template.create_at = timezone.now() + mock_template.update_at = timezone.now() + mock_template.pipeline_tree = { + "activities": { + "node_1": { + "id": "node_1", + "type": "ServiceActivity", + "name": "远程插件节点", + "component": { + "code": "remote_plugin", + "version": "v1.0", + "inputs": { + "plugin_code": {"value": "my_remote_plugin"}, + "plugin_version": {"value": "v2.0"}, + }, + }, + }, + }, + "gateways": {}, + } + + collector._template = mock_template + + nodes = collector._collect_nodes( + pipeline_tree=mock_template.pipeline_tree, + subprocess_stack=[], + is_sub=False, + ) + + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].component_code, "my_remote_plugin") + self.assertEqual(nodes[0].version, "v2.0") + self.assertTrue(nodes[0].is_remote) diff --git a/tests/interface/statistics/test_conf.py b/tests/interface/statistics/test_conf.py new file mode 100644 index 0000000000..e472d7a0e8 --- /dev/null +++ b/tests/interface/statistics/test_conf.py @@ -0,0 +1,96 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import patch + +from django.test import TestCase, override_settings + +from bkflow.statistics.conf import StatisticsConfig + + +class TestStatisticsConfig(TestCase): + """统计配置测试""" + + @patch("env.STATISTICS_ENABLED", True) + def test_is_enabled_true(self): + """测试统计功能启用""" + result = StatisticsConfig.is_enabled() + self.assertTrue(result) + + @patch("env.STATISTICS_ENABLED", False) + def test_is_enabled_false(self): + """测试统计功能禁用""" + result = StatisticsConfig.is_enabled() + self.assertFalse(result) + + @patch("env.BKFLOW_MODULE_CODE", "engine_a") + def test_get_engine_id_from_env(self): + """测试从环境变量获取 Engine ID""" + with patch.object(StatisticsConfig, "get_engine_id") as mock_method: + mock_method.return_value = "engine_a" + result = StatisticsConfig.get_engine_id() + self.assertEqual(result, "engine_a") + + def test_get_db_alias_default(self): + """测试获取默认数据库别名""" + result = StatisticsConfig.get_db_alias() + self.assertEqual(result, "default") + + @override_settings( + DATABASES={ + "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + "statistics": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + } + ) + def test_get_db_alias_with_statistics(self): + """测试配置了 statistics 数据库时的别名""" + result = StatisticsConfig.get_db_alias() + self.assertEqual(result, "statistics") + + @patch("env.STATISTICS_ENABLED", True) + @patch("env.BKFLOW_MODULE_TYPE", "interface") + def test_should_collect_template_stats_interface(self): + """测试 Interface 模块采集模板统计""" + result = StatisticsConfig.should_collect_template_stats() + self.assertTrue(result) + + @patch("env.STATISTICS_ENABLED", True) + @patch("env.BKFLOW_MODULE_TYPE", "engine") + def test_should_collect_template_stats_engine(self): + """测试 Engine 模块不采集模板统计""" + result = StatisticsConfig.should_collect_template_stats() + self.assertFalse(result) + + @patch("env.STATISTICS_INCLUDE_MOCK", True) + def test_include_mock_tasks_true(self): + """测试包含 Mock 任务""" + result = StatisticsConfig.include_mock_tasks() + self.assertTrue(result) + + @patch("env.STATISTICS_INCLUDE_MOCK", False) + def test_include_mock_tasks_false(self): + """测试不包含 Mock 任务""" + result = StatisticsConfig.include_mock_tasks() + self.assertFalse(result) + + @patch("env.STATISTICS_RETENTION_DAYS", 90) + def test_get_retention_days(self): + """测试获取保留天数""" + result = StatisticsConfig.get_retention_days() + self.assertEqual(result, 90) diff --git a/tests/interface/statistics/test_db_router.py b/tests/interface/statistics/test_db_router.py new file mode 100644 index 0000000000..fae6f936ce --- /dev/null +++ b/tests/interface/statistics/test_db_router.py @@ -0,0 +1,120 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from unittest.mock import MagicMock + +from django.test import TestCase, override_settings + +from bkflow.statistics.db_router import StatisticsDBRouter +from bkflow.statistics.models import TaskflowStatistics, TemplateStatistics + + +class TestStatisticsDBRouter(TestCase): + """统计数据库路由器测试""" + + def setUp(self): + self.router = StatisticsDBRouter() + + def test_db_for_read_statistics_model(self): + """测试统计模型读操作路由""" + model = TemplateStatistics + result = self.router.db_for_read(model) + # Should return 'default' since 'statistics' db is not configured + self.assertEqual(result, "default") + + def test_db_for_write_statistics_model(self): + """测试统计模型写操作路由""" + model = TaskflowStatistics + result = self.router.db_for_write(model) + self.assertEqual(result, "default") + + def test_db_for_read_non_statistics_model(self): + """测试非统计模型读操作路由""" + mock_model = MagicMock() + mock_model._meta.app_label = "other_app" + result = self.router.db_for_read(mock_model) + self.assertIsNone(result) + + def test_db_for_write_non_statistics_model(self): + """测试非统计模型写操作路由""" + mock_model = MagicMock() + mock_model._meta.app_label = "other_app" + result = self.router.db_for_write(mock_model) + self.assertIsNone(result) + + def test_allow_relation_same_app(self): + """测试同一 app 内的关联""" + mock_obj1 = MagicMock() + mock_obj1._meta.app_label = "statistics" + mock_obj2 = MagicMock() + mock_obj2._meta.app_label = "statistics" + result = self.router.allow_relation(mock_obj1, mock_obj2) + self.assertTrue(result) + + def test_allow_relation_different_app(self): + """测试不同 app 间的关联""" + mock_obj1 = MagicMock() + mock_obj1._meta.app_label = "statistics" + mock_obj2 = MagicMock() + mock_obj2._meta.app_label = "other_app" + result = self.router.allow_relation(mock_obj1, mock_obj2) + self.assertFalse(result) + + def test_allow_relation_both_non_statistics(self): + """测试两个都是非统计 app 的关联""" + mock_obj1 = MagicMock() + mock_obj1._meta.app_label = "app1" + mock_obj2 = MagicMock() + mock_obj2._meta.app_label = "app2" + result = self.router.allow_relation(mock_obj1, mock_obj2) + self.assertIsNone(result) + + def test_allow_migrate_statistics_app(self): + """测试统计 app 的迁移路由""" + result = self.router.allow_migrate("default", "statistics") + self.assertTrue(result) + + result = self.router.allow_migrate("statistics", "statistics") + self.assertFalse(result) # statistics db doesn't exist, so returns False + + def test_allow_migrate_other_app(self): + """测试其他 app 的迁移路由""" + # 非统计 app 可以在 default 数据库迁移 + result = self.router.allow_migrate("default", "other_app") + self.assertTrue(result) + + # 非统计 app 不应该在 statistics 数据库迁移 + result = self.router.allow_migrate("statistics", "other_app") + self.assertFalse(result) + + @override_settings( + DATABASES={ + "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + "statistics": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, + } + ) + def test_get_db_alias_with_statistics_db(self): + """测试配置了 statistics 数据库时的别名获取""" + result = self.router._get_db_alias() + self.assertEqual(result, "statistics") + + def test_get_db_alias_without_statistics_db(self): + """测试未配置 statistics 数据库时的别名获取""" + result = self.router._get_db_alias() + self.assertEqual(result, "default") diff --git a/tests/interface/statistics/test_models.py b/tests/interface/statistics/test_models.py new file mode 100644 index 0000000000..b820a5bad7 --- /dev/null +++ b/tests/interface/statistics/test_models.py @@ -0,0 +1,152 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import date + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TemplateNodeStatistics, + TemplateStatistics, +) + + +class TestTemplateNodeStatistics(TestCase): + """模板节点统计模型测试""" + + def test_create_template_node_statistics(self): + """测试创建模板节点统计""" + stat = TemplateNodeStatistics.objects.create( + component_code="test_component", + version="v1.0", + is_remote=False, + template_id=1, + space_id=1, + scope_type="biz", + scope_value="2", + node_id="node_1", + node_name="测试节点", + is_sub=False, + subprocess_stack="[]", + template_creator="admin", + template_create_time=timezone.now(), + template_update_time=timezone.now(), + ) + self.assertEqual(stat.component_code, "test_component") + self.assertEqual(stat.template_id, 1) + self.assertFalse(stat.is_remote) + + def test_str_representation(self): + """测试字符串表示""" + stat = TemplateNodeStatistics( + component_code="test_component", + template_id=1, + node_id="node_1", + ) + self.assertEqual(str(stat), "test_component_1_node_1") + + +class TestTemplateStatistics(TestCase): + """模板统计模型测试""" + + def test_create_template_statistics(self): + """测试创建模板统计""" + stat = TemplateStatistics.objects.create( + template_id=1, + space_id=1, + atom_total=5, + subprocess_total=2, + gateways_total=3, + input_count=4, + output_count=2, + template_name="测试模板", + template_creator="admin", + is_enabled=True, + ) + self.assertEqual(stat.atom_total, 5) + self.assertEqual(stat.template_name, "测试模板") + + def test_unique_template_id(self): + """测试模板ID唯一性""" + TemplateStatistics.objects.create( + template_id=1, + space_id=1, + ) + with self.assertRaises(Exception): + TemplateStatistics.objects.create( + template_id=1, + space_id=1, + ) + + +class TestDailyStatisticsSummary(TestCase): + """每日统计汇总模型测试""" + + def test_create_daily_summary(self): + """测试创建每日统计汇总""" + stat = DailyStatisticsSummary.objects.create( + date=date.today(), + space_id=1, + task_created_count=100, + task_success_count=90, + task_failed_count=10, + avg_task_elapsed_time=60.5, + ) + self.assertEqual(stat.task_created_count, 100) + self.assertEqual(stat.task_success_count, 90) + + def test_unique_together(self): + """测试唯一约束""" + today = date.today() + DailyStatisticsSummary.objects.create( + date=today, + space_id=1, + scope_type="biz", + scope_value="2", + ) + with self.assertRaises(Exception): + DailyStatisticsSummary.objects.create( + date=today, + space_id=1, + scope_type="biz", + scope_value="2", + ) + + +class TestPluginExecutionSummary(TestCase): + """插件执行汇总模型测试""" + + def test_create_plugin_summary(self): + """测试创建插件执行汇总""" + stat = PluginExecutionSummary.objects.create( + period_type="day", + period_start=date.today(), + space_id=1, + component_code="test_component", + version="v1.0", + execution_count=100, + success_count=95, + failed_count=5, + avg_elapsed_time=5.5, + ) + self.assertEqual(stat.execution_count, 100) + self.assertEqual(stat.success_count, 95) diff --git a/tests/interface/statistics/test_serializers.py b/tests/interface/statistics/test_serializers.py new file mode 100644 index 0000000000..eededb67b5 --- /dev/null +++ b/tests/interface/statistics/test_serializers.py @@ -0,0 +1,164 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import date + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TaskflowStatistics, + TemplateStatistics, +) +from bkflow.statistics.serializers import ( + DailyStatisticsSummarySerializer, + PluginExecutionSummarySerializer, + PluginRankingSerializer, + StatisticsOverviewSerializer, + TaskflowStatisticsSerializer, + TaskTrendSerializer, + TemplateRankingSerializer, + TemplateStatisticsSerializer, +) + + +class TestStatisticsOverviewSerializer(TestCase): + """统计概览序列化器测试""" + + def test_serialization(self): + """测试序列化""" + data = { + "space_id": 1, + "scope_type": "biz", + "scope_value": "2", + "total_tasks": 100, + "success_tasks": 90, + "failed_tasks": 10, + "success_rate": 90.0, + "total_templates": 50, + "active_templates": 45, + "total_nodes_executed": 1000, + "node_success_rate": 95.0, + "avg_task_elapsed_time": 60.5, + } + serializer = StatisticsOverviewSerializer(data) + self.assertEqual(serializer.data["space_id"], 1) + self.assertEqual(serializer.data["success_rate"], 90.0) + + +class TestTaskTrendSerializer(TestCase): + """任务趋势序列化器测试""" + + def test_serialization(self): + """测试序列化""" + data = { + "date": date.today(), + "task_created_count": 10, + "task_finished_count": 8, + "task_success_count": 7, + "success_rate": 87.5, + } + serializer = TaskTrendSerializer(data) + self.assertEqual(serializer.data["task_created_count"], 10) + + +class TestPluginRankingSerializer(TestCase): + """插件排行序列化器测试""" + + def test_serialization(self): + """测试序列化""" + data = { + "component_code": "test_component", + "version": "v1.0", + "execution_count": 100, + "success_count": 95, + "failed_count": 5, + "success_rate": 95.0, + "avg_elapsed_time": 5.5, + } + serializer = PluginRankingSerializer(data) + self.assertEqual(serializer.data["component_code"], "test_component") + self.assertEqual(serializer.data["success_rate"], 95.0) + + +class TestTemplateRankingSerializer(TestCase): + """模板排行序列化器测试""" + + def test_serialization(self): + """测试序列化""" + data = { + "template_id": 1, + "template_name": "测试模板", + "task_count": 50, + "success_count": 45, + "success_rate": 90.0, + } + serializer = TemplateRankingSerializer(data) + self.assertEqual(serializer.data["template_id"], 1) + self.assertEqual(serializer.data["template_name"], "测试模板") + + +class TestModelSerializers(TestCase): + """模型序列化器测试""" + + def test_template_statistics_serializer(self): + """测试模板统计序列化器""" + stat = TemplateStatistics( + template_id=1, + space_id=1, + atom_total=5, + template_name="测试模板", + ) + serializer = TemplateStatisticsSerializer(stat) + self.assertEqual(serializer.data["template_id"], 1) + + def test_taskflow_statistics_serializer(self): + """测试任务统计序列化器""" + stat = TaskflowStatistics( + task_id=1, + instance_id="instance_001", + space_id=1, + create_time=timezone.now(), + ) + serializer = TaskflowStatisticsSerializer(stat) + self.assertEqual(serializer.data["task_id"], 1) + + def test_daily_summary_serializer(self): + """测试每日汇总序列化器""" + stat = DailyStatisticsSummary( + date=date.today(), + space_id=1, + task_created_count=100, + ) + serializer = DailyStatisticsSummarySerializer(stat) + self.assertEqual(serializer.data["task_created_count"], 100) + + def test_plugin_summary_serializer(self): + """测试插件汇总序列化器""" + stat = PluginExecutionSummary( + period_type="day", + period_start=date.today(), + space_id=1, + component_code="test_component", + execution_count=50, + ) + serializer = PluginExecutionSummarySerializer(stat) + self.assertEqual(serializer.data["execution_count"], 50) diff --git a/tests/interface/statistics/test_tasks.py b/tests/interface/statistics/test_tasks.py new file mode 100644 index 0000000000..c6be5994d2 --- /dev/null +++ b/tests/interface/statistics/test_tasks.py @@ -0,0 +1,156 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import date, timedelta +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + PluginExecutionSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, +) +from bkflow.statistics.tasks import ( + clean_expired_statistics_task, + generate_daily_summary_task, + generate_plugin_summary_task, + template_post_save_statistics_task, +) + + +class TestTemplateStatisticsTasks(TestCase): + """模板统计任务测试""" + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + def test_template_task_disabled(self, mock_is_enabled): + """测试统计功能禁用时跳过采集""" + mock_is_enabled.return_value = False + # Should not raise any exception + template_post_save_statistics_task(template_id=1) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.collectors.TemplateStatisticsCollector.collect") + def test_template_task_enabled(self, mock_collect, mock_is_enabled): + """测试统计功能启用时执行采集""" + mock_is_enabled.return_value = True + mock_collect.return_value = True + template_post_save_statistics_task(template_id=1) + mock_collect.assert_called_once() + + +class TestSummaryTasks(TestCase): + """汇总任务测试（在 Interface 模块执行）""" + + def setUp(self): + """创建测试数据""" + now = timezone.now() + + # 创建任务统计数据 + TaskflowStatistics.objects.create( + task_id=1, + instance_id="instance_001", + space_id=1, + scope_type="biz", + scope_value="2", + create_time=now - timedelta(days=1), + is_started=True, + is_finished=True, + is_success=True, + elapsed_time=60, + ) + TaskflowStatistics.objects.create( + task_id=2, + instance_id="instance_002", + space_id=1, + scope_type="biz", + scope_value="2", + create_time=now - timedelta(days=1), + is_started=True, + is_finished=True, + is_success=False, + elapsed_time=120, + ) + + # 创建节点执行统计数据 + TaskflowExecutedNodeStatistics.objects.create( + component_code="test_component", + version="v1.0", + task_id=1, + instance_id="instance_001", + space_id=1, + scope_type="biz", + scope_value="2", + node_id="node_1", + started_time=now - timedelta(days=1), + status=True, + state="FINISHED", + task_create_time=now - timedelta(days=1), + ) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + def test_daily_summary_task_disabled(self, mock_is_enabled): + """测试统计功能禁用时跳过汇总""" + mock_is_enabled.return_value = False + generate_daily_summary_task() + self.assertEqual(DailyStatisticsSummary.objects.count(), 0) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_daily_summary_task_enabled(self, mock_db_alias, mock_is_enabled): + """测试每日汇总任务""" + mock_is_enabled.return_value = True + mock_db_alias.return_value = "default" + yesterday = (date.today() - timedelta(days=1)).isoformat() + generate_daily_summary_task(target_date=yesterday) + # 检查是否创建了汇总记录 + summaries = DailyStatisticsSummary.objects.all() + self.assertGreaterEqual(summaries.count(), 1) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_plugin_summary_task(self, mock_db_alias, mock_is_enabled): + """测试插件汇总任务""" + mock_is_enabled.return_value = True + mock_db_alias.return_value = "default" + yesterday = (date.today() - timedelta(days=1)).isoformat() + generate_plugin_summary_task(period_type="day", target_date=yesterday) + # 检查是否创建了汇总记录 + summaries = PluginExecutionSummary.objects.all() + self.assertGreaterEqual(summaries.count(), 1) + + @patch("bkflow.statistics.conf.StatisticsConfig.is_enabled") + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + @patch("bkflow.statistics.conf.StatisticsConfig.get_retention_days") + def test_clean_expired_statistics_task(self, mock_retention, mock_db_alias, mock_is_enabled): + """测试清理过期统计数据""" + mock_is_enabled.return_value = True + mock_db_alias.return_value = "default" + mock_retention.return_value = 30 + # 创建过期数据 + old_date = date.today() - timedelta(days=60) + DailyStatisticsSummary.objects.create( + date=old_date, + space_id=1, + ) + clean_expired_statistics_task() + # 过期数据应该被清理 + self.assertEqual(DailyStatisticsSummary.objects.filter(date=old_date).count(), 0) diff --git a/tests/interface/statistics/test_views.py b/tests/interface/statistics/test_views.py new file mode 100644 index 0000000000..052de99488 --- /dev/null +++ b/tests/interface/statistics/test_views.py @@ -0,0 +1,238 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" + +from datetime import date, timedelta +from unittest.mock import patch + +import pytest +from blueapps.account.models import User +from django.utils import timezone +from rest_framework.test import APIRequestFactory, force_authenticate + +from bkflow.statistics.models import ( + DailyStatisticsSummary, + TaskflowExecutedNodeStatistics, + TaskflowStatistics, + TemplateStatistics, +) +from bkflow.statistics.views import SpaceStatisticsViewSet + + +@pytest.mark.django_db +class TestSpaceStatisticsViewSet: + """空间统计 API 测试""" + + def setup_method(self): + """设置测试数据""" + self.factory = APIRequestFactory() + self.user = User.objects.create_superuser(username="testuser", password="password") + now = timezone.now() + + # 创建模板统计 + TemplateStatistics.objects.create( + template_id=1, + space_id=1, + template_name="测试模板1", + atom_total=5, + is_enabled=True, + ) + TemplateStatistics.objects.create( + template_id=2, + space_id=1, + template_name="测试模板2", + atom_total=3, + is_enabled=True, + ) + + # 创建任务统计 + for i in range(10): + TaskflowStatistics.objects.create( + task_id=i + 1, + instance_id=f"instance_{i+1:03d}", + template_id=1 if i < 6 else 2, + space_id=1, + create_time=now - timedelta(days=i % 7), + is_started=True, + is_finished=True, + is_success=(i % 3 != 0), + elapsed_time=60 + i * 10, + ) + + # 创建节点执行统计 + for i in range(20): + TaskflowExecutedNodeStatistics.objects.create( + component_code=f"component_{i % 3}", + version="v1.0", + task_id=(i % 10) + 1, + instance_id=f"instance_{(i % 10)+1:03d}", + space_id=1, + node_id=f"node_{i}", + started_time=now - timedelta(days=i % 7), + status=(i % 4 != 0), + state="FINISHED" if i % 4 != 0 else "FAILED", + elapsed_time=5 + i, + task_create_time=now - timedelta(days=i % 7), + ) + + # 创建每日汇总 + for i in range(7): + DailyStatisticsSummary.objects.create( + date=date.today() - timedelta(days=i), + space_id=1, + task_created_count=10 + i, + task_finished_count=9 + i, + task_success_count=8 + i, + ) + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_overview_without_space_id(self, mock_db_alias): + """测试缺少 space_id 参数""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "overview"}) + request = self.factory.get("/api/statistics/space/overview/") + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 400 + assert "error" in response.data + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_overview_with_space_id(self, mock_db_alias): + """测试获取空间概览""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "overview"}) + request = self.factory.get("/api/statistics/space/overview/", {"space_id": 1}) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + assert response.data["space_id"] == 1 + assert "total_tasks" in response.data + assert "success_rate" in response.data + assert "total_templates" in response.data + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_task_trend(self, mock_db_alias): + """测试获取任务趋势""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "task_trend"}) + request = self.factory.get("/api/statistics/space/task-trend/", {"space_id": 1, "date_range": "7d"}) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + assert isinstance(response.data, list) + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_plugin_ranking(self, mock_db_alias): + """测试获取插件排行""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "plugin_ranking"}) + request = self.factory.get("/api/statistics/space/plugin-ranking/", {"space_id": 1}) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + assert isinstance(response.data, list) + if len(response.data) > 0: + assert "component_code" in response.data[0] + assert "execution_count" in response.data[0] + assert "success_rate" in response.data[0] + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_template_ranking(self, mock_db_alias): + """测试获取模板排行""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "template_ranking"}) + request = self.factory.get("/api/statistics/space/template-ranking/", {"space_id": 1}) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + assert isinstance(response.data, list) + if len(response.data) > 0: + assert "template_id" in response.data[0] + assert "task_count" in response.data[0] + assert "success_rate" in response.data[0] + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_daily_summary(self, mock_db_alias): + """测试获取每日统计汇总""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "daily_summary"}) + request = self.factory.get("/api/statistics/space/daily-summary/", {"space_id": 1, "date_range": "7d"}) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + assert isinstance(response.data, list) + assert len(response.data) > 0 + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_overview_with_scope_filter(self, mock_db_alias): + """测试带范围过滤的概览""" + mock_db_alias.return_value = "default" + + # 创建带 scope 的数据 + TaskflowStatistics.objects.create( + task_id=100, + instance_id="instance_100", + space_id=1, + scope_type="biz", + scope_value="2", + create_time=timezone.now(), + is_finished=True, + is_success=True, + ) + + view = SpaceStatisticsViewSet.as_view({"get": "overview"}) + request = self.factory.get( + "/api/statistics/space/overview/", {"space_id": 1, "scope_type": "biz", "scope_value": "2"} + ) + force_authenticate(request, user=self.user) + + response = view(request) + assert response.status_code == 200 + + @patch("bkflow.statistics.conf.StatisticsConfig.get_db_alias") + def test_plugin_ranking_order_by(self, mock_db_alias): + """测试插件排行排序""" + mock_db_alias.return_value = "default" + + view = SpaceStatisticsViewSet.as_view({"get": "plugin_ranking"}) + + # 按执行次数排序 + request = self.factory.get( + "/api/statistics/space/plugin-ranking/", {"space_id": 1, "order_by": "execution_count"} + ) + force_authenticate(request, user=self.user) + response = view(request) + assert response.status_code == 200 + + # 按成功率排序 + request = self.factory.get("/api/statistics/space/plugin-ranking/", {"space_id": 1, "order_by": "success_rate"}) + force_authenticate(request, user=self.user) + response = view(request) + assert response.status_code == 200 diff --git a/tests/plugins/query/uniform_api/__init__.py b/tests/plugins/query/uniform_api/__init__.py new file mode 100644 index 0000000000..732d2b8d52 --- /dev/null +++ b/tests/plugins/query/uniform_api/__init__.py @@ -0,0 +1,18 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" diff --git a/tests/plugins/query/uniform_api/test_utils.py b/tests/plugins/query/uniform_api/test_utils.py new file mode 100644 index 0000000000..cd1ccf59cc --- /dev/null +++ b/tests/plugins/query/uniform_api/test_utils.py @@ -0,0 +1,158 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" +from unittest.mock import patch + +import pytest +from django.conf import settings + +from bkflow.exceptions import APIRequestError, ValidationError +from bkflow.pipeline_plugins.query.uniform_api.utils import UniformAPIClient + + +class TestUniformAPIClientMethods: + """测试 UniformAPIClient 的方法""" + + def setup_method(self, method): + self.client = UniformAPIClient(from_apigw_check=False) + + def test_request_with_get_method(self): + """测试 GET 请求""" + response = self.client.request(url="http://www.example.com", method="GET", data={"key": "value"}) + assert response is not None + + def test_request_with_post_method(self): + """测试 POST 请求""" + response = self.client.request(url="http://www.example.com", method="POST", data={"key": "value"}) + assert response is not None + + def test_request_with_lowercase_method(self): + """测试小写方法名会被转换为大写""" + response = self.client.request(url="http://www.example.com", method="get", data={"key": "value"}) + assert response is not None + + def test_request_with_custom_headers(self): + """测试自定义headers""" + custom_headers = {"X-Custom-Header": "test_value"} + response = self.client.request( + url="http://www.example.com", + method="GET", + data={}, + headers=custom_headers, + ) + assert response is not None + + def test_request_with_custom_timeout(self): + """测试自定义timeout""" + response = self.client.request(url="http://www.example.com", method="GET", data={}, timeout=60) + assert response is not None + + @patch.object(UniformAPIClient, "check_url_from_apigw") + def test_request_url_apigw_check_fail(self, mock_check): + """测试URL API网关检查失败""" + mock_check.return_value = False + client = UniformAPIClient(from_apigw_check=True) + + with pytest.raises(APIRequestError) as exc_info: + client.request(url="http://invalid.com", method="GET", data={}) + assert "check url from apigw fail" in str(exc_info.value) + + def test_category_list_response_schema_validation(self): + """测试分类列表响应Schema验证""" + # 有效数据 + valid_data = [{"id": "cat1", "name": "Category 1"}, {"id": "cat2", "name": "Category 2"}] + self.client.validate_response_data(valid_data, self.client.UNIFORM_API_CATEGORY_LIST_RESPONSE_DATA_SCHEMA) + + # 无效数据（缺少必需字段） + with pytest.raises(ValidationError): + invalid_data = [{"id": "cat1"}] # 缺少 name + self.client.validate_response_data(invalid_data, self.client.UNIFORM_API_CATEGORY_LIST_RESPONSE_DATA_SCHEMA) + + @patch.object(settings, "BK_APP_CODE", "default_app") + @patch.object(settings, "BK_APP_SECRET", "default_secret") + def test_request_with_default_headers(self): + """测试使用默认headers""" + client = UniformAPIClient(from_apigw_check=False) + response = client.request( + url="http://www.example.com", + method="GET", + data={}, + headers=None, # 使用默认headers + username="test_user", + ) + assert response is not None + + def test_gen_default_apigw_header(self): + """测试生成默认API网关请求头""" + headers = self.client.gen_default_apigw_header( + app_code="test_app", app_secret="test_secret", username="test_user" + ) + assert "X-Bkapi-Authorization" in headers + + def test_method_not_supported(self): + """测试不支持的方法""" + with pytest.raises(APIRequestError) as exc_info: + self.client.request(url="http://www.example.com", method="DELETE", data={}) + assert "method not supported" in str(exc_info.value) + + def test_list_response_schema_validation_missing_apis(self): + """测试列表响应Schema验证 - 缺少apis字段""" + with pytest.raises(ValidationError): + invalid_data = {"total": 10} # 缺少 apis + self.client.validate_response_data(invalid_data, self.client.UNIFORM_API_LIST_RESPONSE_DATA_SCHEMA) + + def test_list_response_schema_validation_valid(self): + """测试列表响应Schema验证 - 有效数据""" + valid_data = { + "total": 1, + "apis": [{"id": "api1", "name": "API 1", "meta_url": "http://example.com/meta"}], + } + self.client.validate_response_data(valid_data, self.client.UNIFORM_API_LIST_RESPONSE_DATA_SCHEMA) + + def test_meta_response_schema_validation_valid(self): + """测试元数据响应Schema验证 - 有效数据""" + valid_data = { + "id": "api1", + "name": "API 1", + "url": "http://example.com/api", + "methods": ["GET", "POST"], + "inputs": [{"name": "param1", "key": "param1"}], + } + self.client.validate_response_data(valid_data, self.client.UNIFORM_API_META_RESPONSE_DATA_SCHEMA) + + def test_meta_response_schema_validation_invalid_method(self): + """测试元数据响应Schema验证 - 无效方法""" + with pytest.raises(ValidationError): + invalid_data = { + "id": "api1", + "name": "API 1", + "url": "http://example.com/api", + "methods": ["DELETE"], # 不支持的方法 + "inputs": [{"name": "param1", "key": "param1"}], + } + self.client.validate_response_data(invalid_data, self.client.UNIFORM_API_META_RESPONSE_DATA_SCHEMA) + + def test_support_methods(self): + """测试支持的方法常量""" + assert "GET" in self.client.SUPPORT_METHODS + assert "POST" in self.client.SUPPORT_METHODS + assert "DELETE" not in self.client.SUPPORT_METHODS + + def test_timeout_constant(self): + """测试超时常量""" + assert self.client.TIMEOUT == 30 diff --git a/tests/plugins/query/uniform_api/test_views.py b/tests/plugins/query/uniform_api/test_views.py new file mode 100644 index 0000000000..293936f525 --- /dev/null +++ b/tests/plugins/query/uniform_api/test_views.py @@ -0,0 +1,165 @@ +""" +TencentBlueKing is pleased to support the open source community by making +蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available. +Copyright (C) 2024 THL A29 Limited, +a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +We undertake not to change the open source license (MIT license) applicable + +to the current version of the project delivered to anyone in the future. +""" +import pytest + +from bkflow.exceptions import ValidationError +from bkflow.pipeline_plugins.query.uniform_api.uniform_api import ( + UniformAPIBaseSerializer, + UniformAPICategorySerializer, + UniformAPIListSerializer, + UniformAPIMetaSerializer, +) + + +class TestUniformAPISerializers: + """测试序列化器""" + + def test_base_serializer_requires_template_or_task_id(self): + """测试基础序列化器需要 template_id 或 task_id""" + serializer = UniformAPIBaseSerializer(data={}) + # 序列化器的 validate 方法会抛出 ValidationError + with pytest.raises(ValidationError) as exc_info: + serializer.is_valid(raise_exception=True) + assert "template_id" in str(exc_info.value) or "task_id" in str(exc_info.value) + + def test_base_serializer_with_template_id(self): + """测试基础序列化器使用 template_id""" + serializer = UniformAPIBaseSerializer(data={"template_id": 1}) + assert serializer.is_valid() is True + + def test_base_serializer_with_task_id(self): + """测试基础序列化器使用 task_id""" + serializer = UniformAPIBaseSerializer(data={"task_id": "123"}) + assert serializer.is_valid() is True + + def test_base_serializer_with_both(self): + """测试基础序列化器同时使用 template_id 和 task_id""" + serializer = UniformAPIBaseSerializer(data={"template_id": 1, "task_id": "123"}) + assert serializer.is_valid() is True + + def test_category_serializer_valid(self): + """测试分类序列化器有效数据""" + serializer = UniformAPICategorySerializer( + data={ + "template_id": 1, + "scope_type": "project", + "scope_value": "p1", + "key": "test", + "api_name": "default", + } + ) + assert serializer.is_valid() is True + + def test_category_serializer_minimal(self): + """测试分类序列化器最小有效数据""" + serializer = UniformAPICategorySerializer(data={"template_id": 1}) + assert serializer.is_valid() is True + + def test_category_serializer_all_optional(self): + """测试分类序列化器所有可选字段""" + serializer = UniformAPICategorySerializer( + data={ + "task_id": "123", + "scope_type": "business", + "scope_value": "b1", + "key": "search_key", + "api_name": "custom_api", + } + ) + assert serializer.is_valid() is True + assert serializer.validated_data["scope_type"] == "business" + assert serializer.validated_data["api_name"] == "custom_api" + + def test_list_serializer_valid(self): + """测试列表序列化器有效数据""" + serializer = UniformAPIListSerializer( + data={ + "template_id": 1, + "limit": 10, + "offset": 0, + "scope_type": "project", + "scope_value": "p1", + "category": "cat1", + "key": "test", + } + ) + assert serializer.is_valid() is True + + def test_list_serializer_defaults(self): + """测试列表序列化器默认值""" + serializer = UniformAPIListSerializer(data={"template_id": 1}) + assert serializer.is_valid() is True + assert serializer.validated_data["limit"] == 50 + assert serializer.validated_data["offset"] == 0 + + def test_list_serializer_custom_pagination(self): + """测试列表序列化器自定义分页""" + serializer = UniformAPIListSerializer(data={"template_id": 1, "limit": 100, "offset": 50}) + assert serializer.is_valid() is True + assert serializer.validated_data["limit"] == 100 + assert serializer.validated_data["offset"] == 50 + + def test_list_serializer_with_api_name(self): + """测试列表序列化器带api_name""" + serializer = UniformAPIListSerializer(data={"template_id": 1, "api_name": "custom_api"}) + assert serializer.is_valid() is True + assert serializer.validated_data["api_name"] == "custom_api" + + def test_list_serializer_with_category(self): + """测试列表序列化器带category""" + serializer = UniformAPIListSerializer(data={"template_id": 1, "category": "system"}) + assert serializer.is_valid() is True + assert serializer.validated_data["category"] == "system" + + def test_meta_serializer_requires_meta_url(self): + """测试元数据序列化器需要 meta_url""" + serializer = UniformAPIMetaSerializer(data={"template_id": 1}) + assert serializer.is_valid() is False + assert "meta_url" in serializer.errors + + def test_meta_serializer_valid(self): + """测试元数据序列化器有效数据""" + serializer = UniformAPIMetaSerializer(data={"template_id": 1, "meta_url": "http://example.com/api/meta"}) + assert serializer.is_valid() is True + + def test_meta_serializer_with_scope(self): + """测试元数据序列化器带scope""" + serializer = UniformAPIMetaSerializer( + data={ + "template_id": 1, + "meta_url": "http://example.com/api/meta", + "scope_type": "project", + "scope_value": "p1", + } + ) + assert serializer.is_valid() is True + assert serializer.validated_data["scope_type"] == "project" + assert serializer.validated_data["scope_value"] == "p1" + + def test_meta_serializer_with_task_id(self): + """测试元数据序列化器使用task_id""" + serializer = UniformAPIMetaSerializer(data={"task_id": "123", "meta_url": "http://example.com/api/meta"}) + assert serializer.is_valid() is True + assert serializer.validated_data["task_id"] == "123" + + def test_list_serializer_with_key(self): + """测试列表序列化器带搜索key""" + serializer = UniformAPIListSerializer(data={"template_id": 1, "key": "search_keyword"}) + assert serializer.is_valid() is True + assert serializer.validated_data["key"] == "search_keyword" From b817d35d99c6eb70af018e85a7952b3be814cb2e Mon Sep 17 00:00:00 2001 From: dengyh Date: Thu, 15 Jan 2026 21:31:37 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=B5=81?= =?UTF-8?q?=E7=A8=8B,=E4=BB=BB=E5=8A=A1=E5=92=8C=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=9A=84=E8=BF=90=E8=90=A5=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20--story=3D130062520?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/unittest.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index b517713501..3c73f3ba10 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -16,9 +16,9 @@ jobs: python-version: [3.9] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -29,7 +29,7 @@ jobs: - name: Setup MySQL uses: shogo82148/actions-setup-mysql@v1 - id: setup-mysql + id: setup-mysql # Add an ID to reference outputs with: mysql-version: '8.0' user: 'test_user' @@ -61,11 +61,3 @@ jobs: sh scripts/run_interface_unit_test.sh echo "run engine unit test" sh scripts/run_engine_unit_test.sh - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: false - verbose: true From 15340aceaeaebaf5afa038fe6b06d295d6c1b39e Mon Sep 17 00:00:00 2001 From: dengyh Date: Fri, 16 Jan 2026 16:01:56 +0800 Subject: [PATCH 3/3] =?UTF-8?q?refactor:=20=E4=BF=AE=E5=A4=8D=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=94=A8=E4=BE=8B=E7=9A=84=E9=97=AE=E9=A2=98=20--stor?= =?UTF-8?q?y=3D130062520?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/interface/statistics/test_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/interface/statistics/test_tasks.py b/tests/interface/statistics/test_tasks.py index c6be5994d2..042969ec82 100644 --- a/tests/interface/statistics/test_tasks.py +++ b/tests/interface/statistics/test_tasks.py @@ -17,7 +17,7 @@ to the current version of the project delivered to anyone in the future. """ -from datetime import date, timedelta +from datetime import timedelta from unittest.mock import patch from django.test import TestCase @@ -119,7 +119,7 @@ def test_daily_summary_task_enabled(self, mock_db_alias, mock_is_enabled): """测试每日汇总任务""" mock_is_enabled.return_value = True mock_db_alias.return_value = "default" - yesterday = (date.today() - timedelta(days=1)).isoformat() + yesterday = (timezone.localdate() - timedelta(days=1)).isoformat() generate_daily_summary_task(target_date=yesterday) # 检查是否创建了汇总记录 summaries = DailyStatisticsSummary.objects.all() @@ -131,7 +131,7 @@ def test_plugin_summary_task(self, mock_db_alias, mock_is_enabled): """测试插件汇总任务""" mock_is_enabled.return_value = True mock_db_alias.return_value = "default" - yesterday = (date.today() - timedelta(days=1)).isoformat() + yesterday = (timezone.localdate() - timedelta(days=1)).isoformat() generate_plugin_summary_task(period_type="day", target_date=yesterday) # 检查是否创建了汇总记录 summaries = PluginExecutionSummary.objects.all() @@ -146,7 +146,7 @@ def test_clean_expired_statistics_task(self, mock_retention, mock_db_alias, mock mock_db_alias.return_value = "default" mock_retention.return_value = 30 # 创建过期数据 - old_date = date.today() - timedelta(days=60) + old_date = timezone.localdate() - timedelta(days=60) DailyStatisticsSummary.objects.create( date=old_date, space_id=1,