Sync 2024-11-22

README.md

@@ -158,8 +158,8 @@ or if you want to use socket:
 
 **version**:
 
-- Use hystax/optscale git tag (eg: 2024101501-public) if you use optscale public version.
-- Use your own tag version if you build your optscale images (eg: latest).
+- Use hystax/optscale git tag (eg: latest) if you use optscale public version.
+- Use your own tag version if you build your optscale images (eg: local).
 
 **please note**: if you use key authentication, you should have the required key (id_rsa) on the machine
 

arcee/arcee_receiver/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.10.2
+aiohttp==3.10.11
 sanic==23.12.1
 sanic-ext==23.12.0
 motor==3.6.0

bulldozer/bulldozer_api/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.10.2
+aiohttp==3.10.11
 sanic==23.12.1
 motor==3.6.0
 pymongo==4.9.1

docker_images/keeper_executor/events.py

@@ -7,6 +7,29 @@ class Events(enum.Enum):
         [],  # parameters
         "INFO"  # event level
     ]
+    N0023 = [
+        'Power schedule {object_name} ({object_id}) created',
+        ['object_name', 'object_id'],
+        "INFO"
+    ]
+    N0024 = [
+        'Power schedule {object_name} ({object_id}) updated',
+        ['object_name', 'object_id'],
+        "INFO"
+    ]
+    N0025 = [
+        'Power schedule {object_name} ({object_id}) deleted',
+        ['object_name', 'object_id'],
+        "INFO"
+    ]
+    N0026 = [
+        'Power schedule {object_name} ({object_id}) processing completed: '
+        '{success_count} resources powered {vm_action}, {error_count} '
+        'resources failed, {excluded_count} resources excluded from schedule',
+        ['object_name', 'object_id', 'success_count', 'error_count',
+         'excluded_count', 'vm_action'],
+        "INFO"
+    ]
     N0027 = [
         'Organization {object_name} ({object_id}) created',
         ['object_name', 'object_id'],

docker_images/keeper_executor/executors/main_events.py

@@ -9,6 +9,10 @@ class MainEventExecutor(BaseEventExecutor):
     @property
     def action_event_map(self):
         return {
+            'power_schedule_created': Events.N0023,
+            'power_schedule_updated': Events.N0024,
+            'power_schedule_deleted': Events.N0025,
+            'power_schedule_processed': Events.N0026,
             'organization_created': Events.N0027,
             'organization_deleted': Events.N0028,
             'organization_updated': Events.N0029,

docker_images/keeper_executor/worker.py

@@ -27,7 +27,7 @@
     'alert.action.added', 'alert.action.removed', 'rule.#', 'environment.#',
     'task.*', 'metric.*', 'run.*', 'platform.*', 'leaderboard_template.*',
     'leaderboard.*', 'dataset.*', 'model.*', 'model_version.*', 'artifact.*',
-    'runset_template.*', 'runset.*', 'runner.*'
+    'runset_template.*', 'runset.*', 'runner.*', 'power_schedule.*'
 ]
 TASK_QUEUE = Queue(QUEUE_NAME, TASK_EXCHANGE, bindings=[
     binding(TASK_EXCHANGE, routing_key=routing_key)

docker_images/power_schedule/tests/test_worker.py

@@ -18,6 +18,7 @@ def setUp(self) -> None:
         patch('docker_images.power_schedule.'
               'worker.PowerScheduleWorker.rest_cl',
               new_callable=PropertyMock).start()
+        self.worker.publish_activities_task = MagicMock()
         self.worker.rest_cl.cloud_account_get = MagicMock(
             return_value=(200, {
                 'id': str(uuid.uuid4()),
@@ -27,6 +28,8 @@ def setUp(self) -> None:
         self.ps_id = str(uuid.uuid4())
         self.valid_ps = {
             'id': self.ps_id,
+            'organization_id': str(uuid.uuid4()),
+            'name': 'my schedule',
             'enabled': True,
             'start_date': 0,
             'end_date': int((datetime.now() + timedelta(days=10)).timestamp()),
@@ -77,6 +80,7 @@ def test_outdated_schedule(self):
         self.assertNotIn(
             'last_run_error',
             self.worker.rest_cl.power_schedule_update.call_args[0][1])
+        self.worker.publish_activities_task.assert_not_called()
 
     def test_schedule_with_zero_end_date(self):
         ps = self.valid_ps.copy()
@@ -130,6 +134,7 @@ def test_disabled_schedule(self):
         self.assertNotIn(
             'last_run_error',
             self.worker.rest_cl.power_schedule_update.call_args[0][1])
+        self.worker.publish_activities_task.assert_not_called()
 
     def test_excluded_resources(self):
         # not active instance
@@ -227,6 +232,8 @@ def test_stop_instance(self):
                 self.worker.rest_cl.power_schedule_update.call_args[0][1][
                     'last_run_error'], None)
             self.worker.rest_cl.power_schedule_update.reset_mock()
+            self.worker.publish_activities_task.assert_called_once()
+            self.worker.publish_activities_task.reset_mock()
 
     def test_start_instance(self):
         self.mongo_cl.restapi.resources.insert_one({
@@ -278,6 +285,8 @@ def test_start_instance(self):
                 self.worker.rest_cl.power_schedule_update.call_args[0][1][
                     'last_run_error'], None)
             self.worker.rest_cl.power_schedule_update.reset_mock()
+            self.worker.publish_activities_task.assert_called_once()
+            self.worker.publish_activities_task.reset_mock()
 
     def test_no_changes(self):
         self.mongo_cl.restapi.resources.insert_one({
@@ -332,6 +341,7 @@ def test_no_changes(self):
                 'last_run_error',
                 self.worker.rest_cl.power_schedule_update.call_args[0][1])
             self.worker.rest_cl.power_schedule_update.reset_mock()
+            self.worker.publish_activities_task.assert_not_called()
 
     def test_no_resources(self):
         ps = self.valid_ps.copy()
@@ -361,6 +371,7 @@ def test_no_resources(self):
         self.assertIn(
             'last_run_error',
             self.worker.rest_cl.power_schedule_update.call_args[0][1])
+        self.worker.publish_activities_task.assert_not_called()
 
     def test_cloud_error(self):
         def raise_exc(*args):

docker_images/power_schedule/worker.py

@@ -13,6 +13,7 @@
 from kombu import Connection
 from kombu.utils.debug import setup_logging
 from kombu import Exchange, Queue
+from kombu.pools import producers
 
 from optscale_client.config_client.client import Client as ConfigClient
 from optscale_client.rest_api_client.client_v2 import Client as RestClient
@@ -22,7 +23,7 @@
 
 from docker_images.power_schedule.utils import is_schedule_outdated
 
-
+ACTIVITIES_EXCHANGE_NAME = 'activities-tasks'
 EXCHANGE_NAME = 'power-schedule'
 QUEUE_NAME = 'power-schedule'
 LOG = get_logger(__name__)
@@ -70,6 +71,27 @@ def get_consumers(self, consumer, channel):
         return [consumer(queues=[TASK_QUEUE], accept=['json'],
                          callbacks=[self.process_task], prefetch_count=3)]
 
+    def publish_activities_task(self, organization_id, object_id, object_type,
+                                object_name, action, meta):
+        task = {
+            'organization_id': organization_id,
+            'object_id': object_id,
+            'object_type': object_type,
+            'object_name': object_name,
+            'action': action,
+            'meta': meta
+        }
+        task_exchange = Exchange(ACTIVITIES_EXCHANGE_NAME, type='topic')
+        with producers[self.connection].acquire(block=True) as producer:
+            producer.publish(
+                task,
+                serializer='json',
+                exchange=task_exchange,
+                declare=[task_exchange],
+                routing_key='.'.join((object_type, action)),
+                retry=True
+            )
+
     @staticmethod
     def _local_time_to_utc(time_str, local_tz):
         return local_tz.localize(datetime.combine(
@@ -192,7 +214,8 @@ def cloud_action(self, action, resource, cloud_type, cloud_adapter):
         self._cloud_action(cloud_adapter, resource_data, action)
         self.result[action] += 1
 
-    def process_resources(self, power_schedule_id, action):
+    def process_resources(self, power_schedule, action):
+        power_schedule_id = power_schedule['id']
         excluded_resources = []
         resources = list(self.mongo_cl.restapi.resources.find(
             {'power_schedule': power_schedule_id}))
@@ -236,6 +259,17 @@ def process_resources(self, power_schedule_id, action):
                     else:
                         LOG.exception('Action %s failed', action)
                         errors.append(exc)
+            meta = {
+                'success_count': (self.result['start_instance'] +
+                                  self.result['stop_instance']),
+                'error_count': self.result['error'],
+                'excluded_count': self.result['excluded'],
+                'vm_action': 'on' if action == 'start_instance' else 'off'
+            }
+            self.publish_activities_task(
+                power_schedule['organization_id'], power_schedule_id,
+                'power_schedule', power_schedule['name'],
+                'power_schedule_processed', meta)
             if errors:
                 raise errors[0]
 
@@ -246,8 +280,7 @@ def process_schedule(self, power_schedule_id):
             return
         required_action = self.get_action(schedule)
         if required_action:
-            self.process_resources(
-                power_schedule_id, required_action)
+            self.process_resources(schedule, required_action)
 
     def process_task(self, body, message):
         self.result = self.default_result().copy()

ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.test.tsx

@@ -18,7 +18,7 @@ it("renders without crashing", () => {
         }}
         onTimeRangeChange={vi.fn}
         isChartLoading={false}
-        isLoading={false}
+        isBreakdownLoading={false}
         archivedRecommendationsBreakdown={vi.fn}
         onBarChartSelect={vi.fn}
       />

ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.tsx

@@ -3,16 +3,19 @@ import { Box, Link, Stack, Typography } from "@mui/material";
 import { FormattedMessage } from "react-intl";
 import { Link as RouterLink } from "react-router-dom";
 import ActionBar from "components/ActionBar";
+import ArchivedResourcesCountBarChart from "components/ArchivedResourcesCountBarChart";
+import BarChartLoader from "components/BarChartLoader";
 import { getBasicRangesSet } from "components/DateRangePicker/defaults";
+import InlineSeverityAlert from "components/InlineSeverityAlert";
 import PageContentWrapper from "components/PageContentWrapper";
 import PanelLoader from "components/PanelLoader";
-import ArchivedRecommendationsBreakdownContainer from "containers/ArchivedRecommendationsBreakdownContainer";
 import ArchivedRecommendationsDetailsContainer from "containers/ArchivedRecommendationsDetailsContainer";
 import RangePickerFormContainer from "containers/RangePickerFormContainer";
 import { RECOMMENDATIONS } from "urls";
 import { isEmpty as isEmptyArray } from "utils/arrays";
 import { DATE_RANGE_TYPE } from "utils/constants";
 import { SPACING_2 } from "utils/layouts";
+import { isEmpty as isEmptyObject } from "utils/objects";
 
 const ArchivedRecommendations = ({
   onBarChartSelect,
@@ -23,20 +26,40 @@ const ArchivedRecommendations = ({
   onDownload,
   isDownloading = false,
   isChartLoading = false,
-  isLoading = false
+  isBreakdownLoading = false
 }) => {
-  const renderArchivedRecommendationsDetails = () => {
-    if (isLoading) {
-      return <PanelLoader />;
+  const renderArchivedResourcesCountBarChart = () => {
+    if (isChartLoading) {
+      return <BarChartLoader />;
     }
-    if (isEmptyArray(archivedRecommendationsBreakdown)) {
+
+    if (Object.values(archivedRecommendationsChartBreakdown).every(isEmptyObject)) {
       return (
-        <Typography align="center">
-          <FormattedMessage id="noRecommendations" />
+        <Typography>
+          <FormattedMessage id="noArchivedRecommendationsAvailable" />
         </Typography>
       );
     }
-    return <ArchivedRecommendationsDetailsContainer archivedRecommendationsBreakdown={archivedRecommendationsBreakdown} />;
+
+    return (
+      <Box>
+        <ArchivedResourcesCountBarChart onSelect={onBarChartSelect} breakdown={archivedRecommendationsChartBreakdown} />
+      </Box>
+    );
+  };
+
+  const renderArchivedRecommendationsDetails = () => {
+    if (isBreakdownLoading) {
+      return <PanelLoader />;
+    }
+    if (isEmptyArray(archivedRecommendationsBreakdown)) {
+      return null;
+    }
+    return (
+      <Box>
+        <ArchivedRecommendationsDetailsContainer archivedRecommendationsBreakdown={archivedRecommendationsBreakdown} />
+      </Box>
+    );
   };
 
   const actionBarDefinition = {
@@ -76,14 +99,11 @@ const ArchivedRecommendations = ({
               definedRanges={getBasicRangesSet()}
             />
           </Box>
+          {renderArchivedResourcesCountBarChart()}
+          {renderArchivedRecommendationsDetails()}
           <Box>
-            <ArchivedRecommendationsBreakdownContainer
-              isLoading={isChartLoading}
-              onBarChartSelect={onBarChartSelect}
-              breakdown={archivedRecommendationsChartBreakdown}
-            />
+            <InlineSeverityAlert messageId="archivedRecommendationsDescription" />
           </Box>
-          <Box>{renderArchivedRecommendationsDetails()}</Box>
         </Stack>
       </PageContentWrapper>
     </>

ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendationsMocked.tsx

@@ -46,7 +46,7 @@ const ArchivedRecommendationsMocked = () => {
       archivedRecommendationsChartBreakdown={archivedRecommendationsChartBreakdown}
       archivedRecommendationsBreakdown={archivedRecommendationsBreakdown}
       isChartLoading={false}
-      isLoading={false}
+      isBreakdownLoading={false}
     />
   );
 };

ngui/ui/src/components/ArchivedResourcesCountBarChart/ArchivedResourcesCountBarChart.tsx

@@ -6,7 +6,6 @@ import KeyValueLabel from "components/KeyValueLabel/KeyValueLabel";
 import { useAllRecommendations } from "hooks/useAllRecommendations";
 import { ARCHIVATION_REASON_MESSAGE_ID } from "utils/constants";
 import { EN_FORMAT_SHORT_YEAR, formatUTC, getEndOfDayInUTCinSeconds, secondsToMilliseconds } from "utils/datetime";
-import { isEmpty as isEmptyObject } from "utils/objects";
 import useStyles from "./ArchivedResourcesCountBarChart.styles";
 
 const NOT_SELECTED = undefined;
@@ -44,10 +43,6 @@ const ChartTooltip = ({ sectionData }) => {
 };
 
 const getChartData = (breakdown) => {
-  if (Object.values(breakdown).every(isEmptyObject)) {
-    return [];
-  }
-
   const chartData = Object.entries(breakdown).map(([date, dateBreakdown]) => ({
     date: formatUTC(date, EN_FORMAT_SHORT_YEAR),
     dateTimestamp: Number(date),

ngui/ui/src/components/Events/Events.tsx

@@ -250,7 +250,13 @@ const Events = ({ eventLevel, includeDebugEvents, descriptionLike, onScroll, app
     const noEvents = isEmpty(events);
 
     if (noEvents) {
-      return isLoading ? <Loader /> : <FormattedMessage id="noEvents" />;
+      return isLoading ? (
+        <Loader />
+      ) : (
+        <Typography>
+          <FormattedMessage id="noEvents" />
+        </Typography>
+      );
     }
 
     return (