From c3b4b1809bebf6d6e72b6eaec58965425af4df62 Mon Sep 17 00:00:00 2001 From: nk-hystax <128669932+nk-hystax@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:43:48 +0300 Subject: [PATCH 1/8] OS-7959. Power schedule events --- docker_images/keeper_executor/events.py | 23 +++++++++++ .../keeper_executor/executors/main_events.py | 4 ++ docker_images/keeper_executor/worker.py | 2 +- .../power_schedule/tests/test_worker.py | 11 +++++ docker_images/power_schedule/worker.py | 41 +++++++++++++++++-- .../controllers/power_schedule.py | 17 ++++++++ 6 files changed, 93 insertions(+), 5 deletions(-) diff --git a/docker_images/keeper_executor/events.py b/docker_images/keeper_executor/events.py index f89a601bc..412988391 100644 --- a/docker_images/keeper_executor/events.py +++ b/docker_images/keeper_executor/events.py @@ -7,6 +7,29 @@ class Events(enum.Enum): [], # parameters "INFO" # event level ] + N0023 = [ + 'Power schedule {object_name} ({object_id}) created', + ['object_name', 'object_id'], + "INFO" + ] + N0024 = [ + 'Power schedule {object_name} ({object_id}) updated', + ['object_name', 'object_id'], + "INFO" + ] + N0025 = [ + 'Power schedule {object_name} ({object_id}) deleted', + ['object_name', 'object_id'], + "INFO" + ] + N0026 = [ + 'Power schedule {object_name} ({object_id}) processing completed: ' + '{success_count} resources powered {vm_action}, {error_count} ' + 'resources failed, {excluded_count} resources excluded from schedule', + ['object_name', 'object_id', 'success_count', 'error_count', + 'excluded_count', 'vm_action'], + "INFO" + ] N0027 = [ 'Organization {object_name} ({object_id}) created', ['object_name', 'object_id'], diff --git a/docker_images/keeper_executor/executors/main_events.py b/docker_images/keeper_executor/executors/main_events.py index 48f77a16c..e47a21748 100644 --- a/docker_images/keeper_executor/executors/main_events.py +++ b/docker_images/keeper_executor/executors/main_events.py @@ -9,6 +9,10 @@ class MainEventExecutor(BaseEventExecutor): @property def action_event_map(self): return { + 'power_schedule_created': Events.N0023, + 'power_schedule_updated': Events.N0024, + 'power_schedule_deleted': Events.N0025, + 'power_schedule_processed': Events.N0026, 'organization_created': Events.N0027, 'organization_deleted': Events.N0028, 'organization_updated': Events.N0029, diff --git a/docker_images/keeper_executor/worker.py b/docker_images/keeper_executor/worker.py index 7a7c6e4c1..36beabd73 100644 --- a/docker_images/keeper_executor/worker.py +++ b/docker_images/keeper_executor/worker.py @@ -27,7 +27,7 @@ 'alert.action.added', 'alert.action.removed', 'rule.#', 'environment.#', 'task.*', 'metric.*', 'run.*', 'platform.*', 'leaderboard_template.*', 'leaderboard.*', 'dataset.*', 'model.*', 'model_version.*', 'artifact.*', - 'runset_template.*', 'runset.*', 'runner.*' + 'runset_template.*', 'runset.*', 'runner.*', 'power_schedule.*' ] TASK_QUEUE = Queue(QUEUE_NAME, TASK_EXCHANGE, bindings=[ binding(TASK_EXCHANGE, routing_key=routing_key) diff --git a/docker_images/power_schedule/tests/test_worker.py b/docker_images/power_schedule/tests/test_worker.py index 4a62e00fa..43c6a1233 100644 --- a/docker_images/power_schedule/tests/test_worker.py +++ b/docker_images/power_schedule/tests/test_worker.py @@ -18,6 +18,7 @@ def setUp(self) -> None: patch('docker_images.power_schedule.' 'worker.PowerScheduleWorker.rest_cl', new_callable=PropertyMock).start() + self.worker.publish_activities_task = MagicMock() self.worker.rest_cl.cloud_account_get = MagicMock( return_value=(200, { 'id': str(uuid.uuid4()), @@ -27,6 +28,8 @@ def setUp(self) -> None: self.ps_id = str(uuid.uuid4()) self.valid_ps = { 'id': self.ps_id, + 'organization_id': str(uuid.uuid4()), + 'name': 'my schedule', 'enabled': True, 'start_date': 0, 'end_date': int((datetime.now() + timedelta(days=10)).timestamp()), @@ -77,6 +80,7 @@ def test_outdated_schedule(self): self.assertNotIn( 'last_run_error', self.worker.rest_cl.power_schedule_update.call_args[0][1]) + self.worker.publish_activities_task.assert_not_called() def test_schedule_with_zero_end_date(self): ps = self.valid_ps.copy() @@ -130,6 +134,7 @@ def test_disabled_schedule(self): self.assertNotIn( 'last_run_error', self.worker.rest_cl.power_schedule_update.call_args[0][1]) + self.worker.publish_activities_task.assert_not_called() def test_excluded_resources(self): # not active instance @@ -227,6 +232,8 @@ def test_stop_instance(self): self.worker.rest_cl.power_schedule_update.call_args[0][1][ 'last_run_error'], None) self.worker.rest_cl.power_schedule_update.reset_mock() + self.worker.publish_activities_task.assert_called_once() + self.worker.publish_activities_task.reset_mock() def test_start_instance(self): self.mongo_cl.restapi.resources.insert_one({ @@ -278,6 +285,8 @@ def test_start_instance(self): self.worker.rest_cl.power_schedule_update.call_args[0][1][ 'last_run_error'], None) self.worker.rest_cl.power_schedule_update.reset_mock() + self.worker.publish_activities_task.assert_called_once() + self.worker.publish_activities_task.reset_mock() def test_no_changes(self): self.mongo_cl.restapi.resources.insert_one({ @@ -332,6 +341,7 @@ def test_no_changes(self): 'last_run_error', self.worker.rest_cl.power_schedule_update.call_args[0][1]) self.worker.rest_cl.power_schedule_update.reset_mock() + self.worker.publish_activities_task.assert_not_called() def test_no_resources(self): ps = self.valid_ps.copy() @@ -361,6 +371,7 @@ def test_no_resources(self): self.assertIn( 'last_run_error', self.worker.rest_cl.power_schedule_update.call_args[0][1]) + self.worker.publish_activities_task.assert_not_called() def test_cloud_error(self): def raise_exc(*args): diff --git a/docker_images/power_schedule/worker.py b/docker_images/power_schedule/worker.py index 86bd5f946..4c8a5ab51 100644 --- a/docker_images/power_schedule/worker.py +++ b/docker_images/power_schedule/worker.py @@ -13,6 +13,7 @@ from kombu import Connection from kombu.utils.debug import setup_logging from kombu import Exchange, Queue +from kombu.pools import producers from optscale_client.config_client.client import Client as ConfigClient from optscale_client.rest_api_client.client_v2 import Client as RestClient @@ -22,7 +23,7 @@ from docker_images.power_schedule.utils import is_schedule_outdated - +ACTIVITIES_EXCHANGE_NAME = 'activities-tasks' EXCHANGE_NAME = 'power-schedule' QUEUE_NAME = 'power-schedule' LOG = get_logger(__name__) @@ -70,6 +71,27 @@ def get_consumers(self, consumer, channel): return [consumer(queues=[TASK_QUEUE], accept=['json'], callbacks=[self.process_task], prefetch_count=3)] + def publish_activities_task(self, organization_id, object_id, object_type, + object_name, action, meta): + task = { + 'organization_id': organization_id, + 'object_id': object_id, + 'object_type': object_type, + 'object_name': object_name, + 'action': action, + 'meta': meta + } + task_exchange = Exchange(ACTIVITIES_EXCHANGE_NAME, type='topic') + with producers[self.connection].acquire(block=True) as producer: + producer.publish( + task, + serializer='json', + exchange=task_exchange, + declare=[task_exchange], + routing_key='.'.join((object_type, action)), + retry=True + ) + @staticmethod def _local_time_to_utc(time_str, local_tz): return local_tz.localize(datetime.combine( @@ -192,7 +214,8 @@ def cloud_action(self, action, resource, cloud_type, cloud_adapter): self._cloud_action(cloud_adapter, resource_data, action) self.result[action] += 1 - def process_resources(self, power_schedule_id, action): + def process_resources(self, power_schedule, action): + power_schedule_id = power_schedule['id'] excluded_resources = [] resources = list(self.mongo_cl.restapi.resources.find( {'power_schedule': power_schedule_id})) @@ -236,6 +259,17 @@ def process_resources(self, power_schedule_id, action): else: LOG.exception('Action %s failed', action) errors.append(exc) + meta = { + 'success_count': (self.result['start_instance'] + + self.result['stop_instance']), + 'error_count': self.result['error'], + 'excluded_count': self.result['excluded'], + 'vm_action': 'on' if action == 'start_instance' else 'off' + } + self.publish_activities_task( + power_schedule['organization_id'], power_schedule_id, + 'power_schedule', power_schedule['name'], + 'power_schedule_processed', meta) if errors: raise errors[0] @@ -246,8 +280,7 @@ def process_schedule(self, power_schedule_id): return required_action = self.get_action(schedule) if required_action: - self.process_resources( - power_schedule_id, required_action) + self.process_resources(schedule, required_action) def process_task(self, body, message): self.result = self.default_result().copy() diff --git a/rest_api/rest_api_server/controllers/power_schedule.py b/rest_api/rest_api_server/controllers/power_schedule.py index 18e829048..8c74048d4 100644 --- a/rest_api/rest_api_server/controllers/power_schedule.py +++ b/rest_api/rest_api_server/controllers/power_schedule.py @@ -63,6 +63,11 @@ def create(self, organization_id: str, **kwargs): power_schedule = super().create( organization_id=organization_id, **kwargs).to_dict() power_schedule['resources_count'] = 0 + self.publish_activities_task( + organization_id, power_schedule["id"], "power_schedule", + "power_schedule_created", {"object_name": power_schedule["name"]}, + "power_schedule.power_schedule_created" + ) return power_schedule def list(self, organization_id: str, **kwargs): @@ -92,6 +97,13 @@ def edit(self, item_id: str, **kwargs): Err.OE0002, [self.model_type.__name__, item_id]) schedule = super().edit(item_id, **kwargs).to_dict() self._set_resources(schedule, show_resources=True) + # not spam events on every schedule run + if set(kwargs) - {'last_eval', 'last_run_error', 'last_run'}: + self.publish_activities_task( + item.organization_id, item_id, "power_schedule", + "power_schedule_updated", {"object_name": item.name}, + "power_schedule.power_schedule_created" + ) return schedule def bulk_action(self, power_schedule_id: str, data: dict): @@ -127,6 +139,11 @@ def delete(self, power_schedule_id): self.resources_collection.update_many( {'power_schedule': power_schedule_id}, {'$unset': {'power_schedule': 1}}) + self.publish_activities_task( + item.organization_id, power_schedule_id, "power_schedule", + "power_schedule_deleted", {"object_name": item.name}, + "power_schedule.power_schedule_deleted" + ) super().delete(power_schedule_id) From 60fdaab11b841d098e5646984906e71397ab44dd Mon Sep 17 00:00:00 2001 From: ek-hystax <33006768+ek-hystax@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:44:06 +0400 Subject: [PATCH 2/8] OS-7987. Add the port to the endpoint_url parameter in the Profiling side modal --- .../components/ProfilingIntegration/ProfilingIntegration.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx index d25028f42..e9c96d09f 100644 --- a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx +++ b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx @@ -143,7 +143,8 @@ const Initialization = ({ const intl = useIntl(); const { onClose } = useContext(ProfilingIntegrationModalContext); - const endpointUrlParameter = isProduction() ? "" : `, endpoint_url="https://${window.location.host}/arcee/v2"`; + + const endpointUrlParameter = isProduction() ? "" : `, endpoint_url="https://${window.location.hostname}:433/arcee/v2"`; const arceeInitUsingContextManager = ( Date: Wed, 20 Nov 2024 16:25:43 +0400 Subject: [PATCH 3/8] OS-4648. Update archived recommendations empty message and add page description --- .../ArchivedRecommendations.test.tsx | 2 +- .../ArchivedRecommendations.tsx | 50 +++++++++++++------ .../ArchivedRecommendationsMocked.tsx | 2 +- .../ArchivedResourcesCountBarChart.tsx | 5 -- ngui/ui/src/components/Events/Events.tsx | 8 ++- ...hivedRecommendationsBreakdownContainer.tsx | 7 --- .../index.ts | 3 -- .../ArchivedRecommendationsContainer.tsx | 2 +- ngui/ui/src/translations/en-US/app.json | 2 + 9 files changed, 47 insertions(+), 34 deletions(-) delete mode 100644 ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/ArchivedRecommendationsBreakdownContainer.tsx delete mode 100644 ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/index.ts diff --git a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.test.tsx b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.test.tsx index da10d2f8e..908a02596 100644 --- a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.test.tsx +++ b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.test.tsx @@ -18,7 +18,7 @@ it("renders without crashing", () => { }} onTimeRangeChange={vi.fn} isChartLoading={false} - isLoading={false} + isBreakdownLoading={false} archivedRecommendationsBreakdown={vi.fn} onBarChartSelect={vi.fn} /> diff --git a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.tsx b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.tsx index 17fe829df..3acfc6bee 100644 --- a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.tsx +++ b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendations.tsx @@ -3,16 +3,19 @@ import { Box, Link, Stack, Typography } from "@mui/material"; import { FormattedMessage } from "react-intl"; import { Link as RouterLink } from "react-router-dom"; import ActionBar from "components/ActionBar"; +import ArchivedResourcesCountBarChart from "components/ArchivedResourcesCountBarChart"; +import BarChartLoader from "components/BarChartLoader"; import { getBasicRangesSet } from "components/DateRangePicker/defaults"; +import InlineSeverityAlert from "components/InlineSeverityAlert"; import PageContentWrapper from "components/PageContentWrapper"; import PanelLoader from "components/PanelLoader"; -import ArchivedRecommendationsBreakdownContainer from "containers/ArchivedRecommendationsBreakdownContainer"; import ArchivedRecommendationsDetailsContainer from "containers/ArchivedRecommendationsDetailsContainer"; import RangePickerFormContainer from "containers/RangePickerFormContainer"; import { RECOMMENDATIONS } from "urls"; import { isEmpty as isEmptyArray } from "utils/arrays"; import { DATE_RANGE_TYPE } from "utils/constants"; import { SPACING_2 } from "utils/layouts"; +import { isEmpty as isEmptyObject } from "utils/objects"; const ArchivedRecommendations = ({ onBarChartSelect, @@ -23,20 +26,40 @@ const ArchivedRecommendations = ({ onDownload, isDownloading = false, isChartLoading = false, - isLoading = false + isBreakdownLoading = false }) => { - const renderArchivedRecommendationsDetails = () => { - if (isLoading) { - return ; + const renderArchivedResourcesCountBarChart = () => { + if (isChartLoading) { + return ; } - if (isEmptyArray(archivedRecommendationsBreakdown)) { + + if (Object.values(archivedRecommendationsChartBreakdown).every(isEmptyObject)) { return ( - - + + ); } - return ; + + return ( + + + + ); + }; + + const renderArchivedRecommendationsDetails = () => { + if (isBreakdownLoading) { + return ; + } + if (isEmptyArray(archivedRecommendationsBreakdown)) { + return null; + } + return ( + + + + ); }; const actionBarDefinition = { @@ -76,14 +99,11 @@ const ArchivedRecommendations = ({ definedRanges={getBasicRangesSet()} /> + {renderArchivedResourcesCountBarChart()} + {renderArchivedRecommendationsDetails()} - + - {renderArchivedRecommendationsDetails()} diff --git a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendationsMocked.tsx b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendationsMocked.tsx index 745fa2605..84efcf3d2 100644 --- a/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendationsMocked.tsx +++ b/ngui/ui/src/components/ArchivedRecommendations/ArchivedRecommendationsMocked.tsx @@ -46,7 +46,7 @@ const ArchivedRecommendationsMocked = () => { archivedRecommendationsChartBreakdown={archivedRecommendationsChartBreakdown} archivedRecommendationsBreakdown={archivedRecommendationsBreakdown} isChartLoading={false} - isLoading={false} + isBreakdownLoading={false} /> ); }; diff --git a/ngui/ui/src/components/ArchivedResourcesCountBarChart/ArchivedResourcesCountBarChart.tsx b/ngui/ui/src/components/ArchivedResourcesCountBarChart/ArchivedResourcesCountBarChart.tsx index 8cb157046..150f5a0f2 100644 --- a/ngui/ui/src/components/ArchivedResourcesCountBarChart/ArchivedResourcesCountBarChart.tsx +++ b/ngui/ui/src/components/ArchivedResourcesCountBarChart/ArchivedResourcesCountBarChart.tsx @@ -6,7 +6,6 @@ import KeyValueLabel from "components/KeyValueLabel/KeyValueLabel"; import { useAllRecommendations } from "hooks/useAllRecommendations"; import { ARCHIVATION_REASON_MESSAGE_ID } from "utils/constants"; import { EN_FORMAT_SHORT_YEAR, formatUTC, getEndOfDayInUTCinSeconds, secondsToMilliseconds } from "utils/datetime"; -import { isEmpty as isEmptyObject } from "utils/objects"; import useStyles from "./ArchivedResourcesCountBarChart.styles"; const NOT_SELECTED = undefined; @@ -44,10 +43,6 @@ const ChartTooltip = ({ sectionData }) => { }; const getChartData = (breakdown) => { - if (Object.values(breakdown).every(isEmptyObject)) { - return []; - } - const chartData = Object.entries(breakdown).map(([date, dateBreakdown]) => ({ date: formatUTC(date, EN_FORMAT_SHORT_YEAR), dateTimestamp: Number(date), diff --git a/ngui/ui/src/components/Events/Events.tsx b/ngui/ui/src/components/Events/Events.tsx index a57d67526..482e77fec 100644 --- a/ngui/ui/src/components/Events/Events.tsx +++ b/ngui/ui/src/components/Events/Events.tsx @@ -250,7 +250,13 @@ const Events = ({ eventLevel, includeDebugEvents, descriptionLike, onScroll, app const noEvents = isEmpty(events); if (noEvents) { - return isLoading ? : ; + return isLoading ? ( + + ) : ( + + + + ); } return ( diff --git a/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/ArchivedRecommendationsBreakdownContainer.tsx b/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/ArchivedRecommendationsBreakdownContainer.tsx deleted file mode 100644 index 05d1f6141..000000000 --- a/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/ArchivedRecommendationsBreakdownContainer.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import ArchivedResourcesCountBarChart from "components/ArchivedResourcesCountBarChart"; -import BarChartLoader from "components/BarChartLoader"; - -const ArchivedRecommendationsBreakdownContainer = ({ isLoading, breakdown, onBarChartSelect }) => - isLoading ? : ; - -export default ArchivedRecommendationsBreakdownContainer; diff --git a/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/index.ts b/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/index.ts deleted file mode 100644 index 8940c1d0e..000000000 --- a/ngui/ui/src/containers/ArchivedRecommendationsBreakdownContainer/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import ArchivedRecommendationsBreakdownContainer from "./ArchivedRecommendationsBreakdownContainer"; - -export default ArchivedRecommendationsBreakdownContainer; diff --git a/ngui/ui/src/containers/ArchivedRecommendationsContainer/ArchivedRecommendationsContainer.tsx b/ngui/ui/src/containers/ArchivedRecommendationsContainer/ArchivedRecommendationsContainer.tsx index d8993a55c..a5294e4ef 100644 --- a/ngui/ui/src/containers/ArchivedRecommendationsContainer/ArchivedRecommendationsContainer.tsx +++ b/ngui/ui/src/containers/ArchivedRecommendationsContainer/ArchivedRecommendationsContainer.tsx @@ -79,7 +79,7 @@ const ArchivedRecommendationsContainer = () => { onDownload={() => onDownload(dateRange)} isChartLoading={isChartLoading} isDownloading={isDownloading} - isLoading={isLoading} + isBreakdownLoading={isLoading} /> ); }; diff --git a/ngui/ui/src/translations/en-US/app.json b/ngui/ui/src/translations/en-US/app.json index b54b57abe..b3e1eb761 100644 --- a/ngui/ui/src/translations/en-US/app.json +++ b/ngui/ui/src/translations/en-US/app.json @@ -152,6 +152,7 @@ "archived": "Archived", "archivedAt": "Archived at", "archivedRecommendations": "Archived Recommendations", + "archivedRecommendationsDescription": "Recommendations are archived when they are no longer active, either because they have been applied, the associated resource has been removed, or they have become irrelevant due to changes in resource properties.", "artifacts": "Artifacts", "assetPool": "Asset pool", "assign": "Assign", @@ -1293,6 +1294,7 @@ "noActiveRecommendationsAvailable": "No active recommendations available", "noAnomalyPolicies": "No anomaly policies", "noApplications": "No applications", + "noArchivedRecommendationsAvailable": "No archived recommendations available.", "noArtifacts": "No artifacts", "noAutomaticResourceAssignmentRules": "No automatic resource assignment rules", "noBIExports": "No Business Intelligence exports", From 046c2f3d3fc0cfb855a00231cd85ffd9a20124d1 Mon Sep 17 00:00:00 2001 From: nk-hystax <128669932+nk-hystax@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:40:53 +0300 Subject: [PATCH 4/8] OS-8003. Updated aiohttp version --- arcee/arcee_receiver/requirements.txt | 2 +- bulldozer/bulldozer_api/requirements.txt | 2 +- optscale_client/aconfig_cl/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arcee/arcee_receiver/requirements.txt b/arcee/arcee_receiver/requirements.txt index 2af97bba5..8ff3aaf85 100644 --- a/arcee/arcee_receiver/requirements.txt +++ b/arcee/arcee_receiver/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.10.2 +aiohttp==3.10.11 sanic==23.12.1 sanic-ext==23.12.0 motor==3.6.0 diff --git a/bulldozer/bulldozer_api/requirements.txt b/bulldozer/bulldozer_api/requirements.txt index 0a14cbc5e..bda4c50e6 100644 --- a/bulldozer/bulldozer_api/requirements.txt +++ b/bulldozer/bulldozer_api/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.10.2 +aiohttp==3.10.11 sanic==23.12.1 motor==3.6.0 pymongo==4.9.1 diff --git a/optscale_client/aconfig_cl/setup.py b/optscale_client/aconfig_cl/setup.py index b99199c54..8a003c783 100644 --- a/optscale_client/aconfig_cl/setup.py +++ b/optscale_client/aconfig_cl/setup.py @@ -9,6 +9,6 @@ url='http://hystax.com', author_email='info@hystax.com', package_dir={'aconfig_cl': ''}, - install_requires=['aiohttp==3.10.2'], + install_requires=['aiohttp==3.10.11'], packages=['aconfig_cl'] ) From 850cd7bae4bb31114882324ce82e9779146d8d27 Mon Sep 17 00:00:00 2001 From: ek-hystax <33006768+ek-hystax@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:50:35 +0400 Subject: [PATCH 5/8] OS-8007. Add GCP filter support for InstancesForShutdown and ShortLivingInstances recommendations --- .../recommendations/InstancesForShutdown.tsx | 4 ++-- .../recommendations/ShortLivingInstances.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx index 8ad9a43d9..3e9b41463 100644 --- a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx +++ b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx @@ -10,7 +10,7 @@ import { NEBIUS_SERVICE } from "hooks/useRecommendationServices"; import { resource, resourceLocation, poolAndOwner, possibleShutdownPeriods, savings } from "utils/columns"; -import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, NEBIUS } from "utils/constants"; +import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, GCP_CNR, NEBIUS } from "utils/constants"; import BaseRecommendation, { CATEGORY_COST } from "./BaseRecommendation"; const columns = [ @@ -44,7 +44,7 @@ class InstancesForShutdown extends BaseRecommendation { services = [AWS_EC2, AWS_RDS, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, ALIBABA_ECS, NEBIUS_SERVICE]; - appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS]; + appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS, GCP_CNR]; categories = [CATEGORY_COST]; diff --git a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx index 910701a72..2db657c2b 100644 --- a/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx +++ b/ngui/ui/src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx @@ -5,7 +5,7 @@ import ShortLivingInstancesModal from "components/SideModalManager/SideModals/re import TextWithDataTestId from "components/TextWithDataTestId"; import { ALIBABA_ECS, AWS_EC2, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, NEBIUS_SERVICE } from "hooks/useRecommendationServices"; import { detectedAt, possibleMonthlySavings, resource, resourceLocation } from "utils/columns"; -import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, NEBIUS } from "utils/constants"; +import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, GCP_CNR, NEBIUS } from "utils/constants"; import BaseRecommendation, { CATEGORY_COST } from "./BaseRecommendation"; const columns = [ @@ -51,7 +51,7 @@ class ShortLivingInstances extends BaseRecommendation { services = [AWS_EC2, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, ALIBABA_ECS, NEBIUS_SERVICE]; - appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS]; + appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS, GCP_CNR]; categories = [CATEGORY_COST]; From c4b887566dd39e8653d23f75d20dfa25cb825d24 Mon Sep 17 00:00:00 2001 From: ek-hystax <33006768+ek-hystax@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:51:18 +0400 Subject: [PATCH 6/8] OS-8011. Update Profiling integration side modal --- .../ProfilingIntegration.tsx | 149 ++++++++++++++---- ngui/ui/src/translations/en-US/app.json | 9 +- 2 files changed, 128 insertions(+), 30 deletions(-) diff --git a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx index e9c96d09f..b488c7710 100644 --- a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx +++ b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx @@ -116,7 +116,7 @@ const Installation = () => ( - + ); @@ -148,13 +148,18 @@ const Initialization = ({ const arceeInitUsingContextManager = ( ); const arceeInitUsingFunctionCall = ( - +
  • {isLoading ? ( ) : ( - {profilingToken}} - isBoldValue={false} - /> +
    + {chunks} + }} + /> + {profilingToken}} + isBoldValue={false} + /> +
    )}
  • + {chunks} + }} + /> {taskKey ? (
    • +
  • + {chunks} + }} + /> +
    • +
  • + {chunks} + }} + /> +
    • +
  • + {chunks} + }} + /> +
    • +
  • + {chunks} + }} + /> +
{isLoading ? {arceeInitUsingContextManager} : arceeInitUsingContextManager} + + - + + + + + + + + + + + + + + + + @@ -283,7 +366,7 @@ const SendingMetrics = () => ( } }} parameterMessageIds={["mlProfilingIntegration.sendingMetrics.parameters.1.data"]} - method={`arcee.send({"metric_key_1": value_1, "metric_key_2": value_2})`} + method={`arcee.send({"YOUR-METRIC-1-KEY": YOUR_METRIC_1_VALUE, "YOUR-METRIC-2-KEY": YOUR_METRIC_2_VALUE})`} example={`arcee.send({ "accuracy": 71.44, "loss": 0.37 })`} /> @@ -307,7 +390,7 @@ const AddingHyperparameters = () => ( "mlProfilingIntegration.addingHyperparameters.parameters.1.key", "mlProfilingIntegration.addingHyperparameters.parameters.2.value" ]} - method={`arcee.hyperparam(key, value)`} + method={`arcee.hyperparam(key="YOUR-PARAM-KEY", value=YOUR_PARAM_VALUE)`} example={`arcee.hyperparam("EPOCHS", 100)`} /> @@ -324,7 +407,7 @@ const TaggingTaskRun = () => ( "mlProfilingIntegration.taggingTaskRun.parameters.1.key", "mlProfilingIntegration.taggingTaskRun.parameters.2.value" ]} - method={`arcee.tag(key, value)`} + method={`arcee.tag(key="YOUR-TAG-KEY", value=YOUR_TAG_VALUE)`} example={`arcee.tag("Algorithm", "Linear Learn Algorithm")`} /> @@ -338,7 +421,7 @@ const AddingMilestone = () => ( id: "mlProfilingIntegration.addingMilestone.description" }} parameterMessageIds={["mlProfilingIntegration.addingMilestone.parameters.1.name"]} - method={`arcee.milestone(name)`} + method={`arcee.milestone(name="YOUR-MILESTONE-NAME")`} example={`arcee.milestone("Download training data")`} /> @@ -352,7 +435,7 @@ const AddingStage = () => ( id: "mlProfilingIntegration.addingStage.description" }} parameterMessageIds={["mlProfilingIntegration.addingStage.parameters.1.name"]} - method={`arcee.stage(name)`} + method={`arcee.stage(name="YOUR-STAGE-NAME")`} example={`arcee.stage("preparing")`} /> @@ -371,7 +454,10 @@ const LoggingDatasets = () => ( "mlProfilingIntegration.loggingDataset.parameters.3.description", "mlProfilingIntegration.loggingDataset.parameters.4.labels" ]} - method={`arcee.dataset(path, name, description, labels)`} + method={`arcee.dataset(path="YOUR-DATASET-PATH", + name="YOUR-DATASET-NAME", + description="YOUR-DATASET-DESCRIPTION", + labels=["YOUR-DATASET-LABEL-1", "YOUR-DATASET-LABEL-2"])`} example={`arcee.dataset("https://s3/ml-bucket/datasets/training_dataset.csv", name="Training dataset", description="Training dataset (100k rows)", @@ -391,7 +477,7 @@ const CreatingModels = () => ( "mlProfilingIntegration.creatingModels.parameters.1.key", "mlProfilingIntegration.creatingModels.parameters.2.path" ]} - method={`arcee.model(key, path)`} + method={`arcee.model(key="YOUR-MODEL-KEY", path="YOUR-MODEL-PATH")`} example={`arcee.model("my_model", "/home/user/my_model")`} /> @@ -405,7 +491,7 @@ const SettingModelVersion = () => ( id: "mlProfilingIntegration.settingModelVersion.description" }} parameterMessageIds={["mlProfilingIntegration.settingModelVersion.parameters.1.version"]} - method={`arcee.model_version(version)`} + method={`arcee.model_version(version="YOUR-MODEL-VERSION")`} example={`arcee.model_version("1.2.3-release")`} /> @@ -419,7 +505,7 @@ const SettingModelVersionAlias = () => ( id: "mlProfilingIntegration.settingModelVersionAlias.description" }} parameterMessageIds={["mlProfilingIntegration.settingModelVersionAlias.parameters.1.alias"]} - method={`arcee.model_version_alias(alias)`} + method={`arcee.model_version_alias(alias="YOUR-MODEL-VERSION-ALIAS")`} example={`arcee.model_version_alias("winner")`} /> @@ -436,7 +522,7 @@ const SettingModelVersionTag = () => ( "mlProfilingIntegration.settingModelVersionTag.parameters.1.key", "mlProfilingIntegration.settingModelVersionTag.parameters.2.value" ]} - method={`arcee.model_version_tag(key, value)`} + method={`arcee.model_version_tag(key="YOUR-MODEL-VERSION-TAG-KEY", value=YOUR_MODEL_VERSION_TAG_VALUE)`} example={`arcee.model_version_tag("env", "staging demo")`} /> @@ -455,7 +541,10 @@ const CreatingArtifacts = () => ( "mlProfilingIntegration.creatingArtifacts.parameters.3.description", "mlProfilingIntegration.creatingArtifacts.parameters.4.tags" ]} - method={`arcee.artifact(path, name, description, tags)`} + method={`arcee.artifact(path="YOUR-ARTIFACT-PATH", + name="YOUR-ARTIFACT-NAME", + description="YOUR-ARTIFACT-DESCRIPTION", + tags={"YOUR-ARTIFACT-TAG-KEY": YOUR_ARTIFACT_TAG_VALUE})`} example={`arcee.artifact("https://s3/ml-bucket/artifacts/AccuracyChart.png", name="Accuracy line chart", description="The dependence of accuracy on the time", @@ -476,7 +565,9 @@ const SettingArtifactTag = () => ( "mlProfilingIntegration.settingArtifactTag.parameters.2.key", "mlProfilingIntegration.settingArtifactTag.parameters.3.value" ]} - method={`arcee.artifact_tag(path, key, value)`} + method={`arcee.artifact_tag(path="YOUR-ARTIFACT-PATH", + key="YOUR-ARTIFACT-TAG-KEY", + value=YOUR_ARTIFACT_TAG_VALUE)`} example={`arcee.artifact_tag("https://s3/ml-bucket/artifacts/AccuracyChart.png", "env", "staging demo")`} /> diff --git a/ngui/ui/src/translations/en-US/app.json b/ngui/ui/src/translations/en-US/app.json index b3e1eb761..b70bef3cb 100644 --- a/ngui/ui/src/translations/en-US/app.json +++ b/ngui/ui/src/translations/en-US/app.json @@ -1182,9 +1182,16 @@ "mlProfilingIntegration.initialization.alternativeInit": "Alternatively, to get more control over error catching and execution finishing, you can initialize the collector using a corresponding method. Note that this method will require you to manually handle errors or terminate arcee execution using the error and finish methods.", "mlProfilingIntegration.initialization.arceeDaemonProcess": "Arcee daemon process periodically sends hardware and process data. The default heartbeat period is 1 second. However, arcee can be initialized with a custom period", "mlProfilingIntegration.initialization.customEndpointAdnSslChecks": "To use a custom endpoint and enable/disable SSL checks (enable self-signed SSL certificates support)", - "mlProfilingIntegration.initialization.description": "To initialize the arcee collector, you need to provide a profiling token and a task key for which you want to collect data", + "mlProfilingIntegration.initialization.description": "To initialize the arcee collector use the init method with the following parameters:", + "mlProfilingIntegration.initialization.forCustomOptScaleDeployment": "For custom OptScale deployments", "mlProfilingIntegration.initialization.initCollectorUsingContextManager": "To initialize the collector using a context manager, use the following code snippet", "mlProfilingIntegration.initialization.initCollectorUsingContextManagerDescription": "This method automatically handles error catching and terminates arcee execution.", + "mlProfilingIntegration.initialization.parameters.1.token": "token (str, required): profiling token", + "mlProfilingIntegration.initialization.parameters.2.task_key": "task_key (str, required): task key for which you want to collect data", + "mlProfilingIntegration.initialization.parameters.3.run_name": "run_name (str, optional): run name", + "mlProfilingIntegration.initialization.parameters.4.endpoint_url": "endpoint_url (str, optional): custom OptScale endpoint (default is https://my.optscale.com/arcee/v2)", + "mlProfilingIntegration.initialization.parameters.5.ssl": "ssl (bool, optional): enable/disable SSL checks (self-signed SSL certificates support)", + "mlProfilingIntegration.initialization.parameters.6.period": "period (int, optional): arcee daemon process heartbeat period in seconds (default is 1)", "mlProfilingIntegration.initialization.profilingToken": "Profiling token", "mlProfilingIntegration.initialization.taskKey": "Task key", "mlProfilingIntegration.initialization.taskKeyCanBeFound": "The task key can be found on the Tasks page", From ccc2e558bd44b1be48991b71cc2f57d5be745a23 Mon Sep 17 00:00:00 2001 From: sd-hystax <110374605+sd-hystax@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:56:13 +0300 Subject: [PATCH 7/8] OS-7958. Ability to update cluster with the latest release ## Description Ability to update cluster with the latest release ## Related issue number OS-7958 ## Checklist * [ ] The pull request title is a good summary of the changes * [ ] Unit tests for the changes exist * [ ] New and existing unit tests pass locally --- README.md | 4 ++-- optscale-deploy/runkube.py | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 920074a39..10eac9f39 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,8 @@ or if you want to use socket: **version**: -- Use hystax/optscale git tag (eg: 2024101501-public) if you use optscale public version. -- Use your own tag version if you build your optscale images (eg: latest). +- Use hystax/optscale git tag (eg: latest) if you use optscale public version. +- Use your own tag version if you build your optscale images (eg: local). **please note**: if you use key authentication, you should have the required key (id_rsa) on the machine diff --git a/optscale-deploy/runkube.py b/optscale-deploy/runkube.py index 81493530c..837b556ce 100755 --- a/optscale-deploy/runkube.py +++ b/optscale-deploy/runkube.py @@ -18,14 +18,16 @@ from docker import DockerClient from kubernetes.stream.ws_client import ERROR_CHANNEL from docker.errors import ImageNotFound - -DESCRIPTION = "Script to deploy OptScale on k8s. " \ - "See deployment instructions at https://github.com/hystax/optscale" +REPOSITORY = 'hystax/optscale' +DESCRIPTION = f"Script to deploy OptScale on k8s. " \ + f"See deployment instructions at https://github.com/{REPOSITORY}" HELM_DELETE_CMD = 'helm delete --purge {release}' HELM_UPDATE_CMD = 'helm upgrade --install {overlays} {release} {chart}' GET_FAKE_CERT_CMD = 'cat /ingress-controller/ssl/default-defaultcert.pem' HELM_LIST_CMD = 'helm list -a' HELM_GET_VALUES_CMD = 'helm get values {name}' +GET_LATEST_TAG_CMD = f"curl https://api.github.com/repos/{REPOSITORY}/releases" \ + f" | jq -r '.[0].tag_name'" TEMP_DIR = 'tmp' BASE_OVERLAY = os.path.join(TEMP_DIR, 'base_overlay') ORIGINAL_OVERLAY = os.path.join(TEMP_DIR, 'original_overlay') @@ -34,6 +36,7 @@ OPTSCALE_K8S_NAMESPACE = 'default' COMPONENTS_FILE = 'components.yaml' LOCAL_TAG = 'local' +LATEST_TAG = 'latest' LOG = logging.getLogger(__name__) @@ -321,8 +324,15 @@ def get_old_overlay_list_for_update(self): except IndexError: LOG.info('etcd pod not found') + def check_version(self): + if self.version.lower() == LATEST_TAG: + self.version = subprocess.check_output( + GET_LATEST_TAG_CMD, shell=True).decode("utf-8").rstrip() + LOG.info('Latest release tag: %s' % self.version) + def start(self, check, update): self.check_releases(update) + self.check_version() for node in self.get_node_ips(): docker_cl = self.get_docker_cl(node) if not self.no_pull: @@ -382,7 +392,8 @@ def delete(self): description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('name', help='Release name for helm and log separation') - parser.add_argument('version', help='OptScale version') + parser.add_argument('version', help='OptScale version. Use `latest` to ' + 'update on latest release version') parser.add_argument('-c', '--config', help='Path to kube config file') action_group = parser.add_mutually_exclusive_group(required=False) action_group.add_argument('-r', '--restart', action='store_true', From ea7b1b2564f33d4673e96f47e3bdaac175dac692 Mon Sep 17 00:00:00 2001 From: ek-hystax <33006768+ek-hystax@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:11:53 +0400 Subject: [PATCH 8/8] OS-8011. Update profiling integration examples --- .../ProfilingIntegration.tsx | 70 ++++++------------- ngui/ui/src/translations/en-US/app.json | 3 - 2 files changed, 23 insertions(+), 50 deletions(-) diff --git a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx index b488c7710..9d79438ed 100644 --- a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx +++ b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx @@ -131,6 +131,10 @@ const Import = () => ( ); +const ENDPOINT_URL = `https://${window.location.hostname}:443/arcee/v2` as const; + +const ENDPOINT_URL_PARAMETER = `endpoint_url="${ENDPOINT_URL}"` as const; + const Initialization = ({ profilingToken, taskKey, @@ -144,8 +148,6 @@ const Initialization = ({ const { onClose } = useContext(ProfilingIntegrationModalContext); - const endpointUrlParameter = isProduction() ? "" : `, endpoint_url="https://${window.location.hostname}:433/arcee/v2"`; - const arceeInitUsingContextManager = ( - - - - - - - - - - + ) : ( + + /> + )} @@ -312,7 +312,7 @@ arcee.error() }} /> - + {isLoading ? {arceeInitUsingFunctionCall} : arceeInitUsingFunctionCall} - - - - - - - - - - - - - - ); }; @@ -366,7 +342,7 @@ const SendingMetrics = () => ( } }} parameterMessageIds={["mlProfilingIntegration.sendingMetrics.parameters.1.data"]} - method={`arcee.send({"YOUR-METRIC-1-KEY": YOUR_METRIC_1_VALUE, "YOUR-METRIC-2-KEY": YOUR_METRIC_2_VALUE})`} + method={`arcee.send({ "YOUR-METRIC-1-KEY": YOUR_METRIC_1_VALUE, "YOUR-METRIC-2-KEY": YOUR_METRIC_2_VALUE })`} example={`arcee.send({ "accuracy": 71.44, "loss": 0.37 })`} /> @@ -568,7 +544,7 @@ const SettingArtifactTag = () => ( method={`arcee.artifact_tag(path="YOUR-ARTIFACT-PATH", key="YOUR-ARTIFACT-TAG-KEY", value=YOUR_ARTIFACT_TAG_VALUE)`} - example={`arcee.artifact_tag("https://s3/ml-bucket/artifacts/AccuracyChart.png", + example={`arcee.artifact_tag("https://s3/ml-bucket/artifacts/AccuracyChart.png", "env", "staging demo")`} /> diff --git a/ngui/ui/src/translations/en-US/app.json b/ngui/ui/src/translations/en-US/app.json index b70bef3cb..65d79b0da 100644 --- a/ngui/ui/src/translations/en-US/app.json +++ b/ngui/ui/src/translations/en-US/app.json @@ -1180,10 +1180,7 @@ "mlProfilingIntegration.import.description": "Import the optscale_arcee module into your code as follows", "mlProfilingIntegration.import.title": "Import", "mlProfilingIntegration.initialization.alternativeInit": "Alternatively, to get more control over error catching and execution finishing, you can initialize the collector using a corresponding method. Note that this method will require you to manually handle errors or terminate arcee execution using the error and finish methods.", - "mlProfilingIntegration.initialization.arceeDaemonProcess": "Arcee daemon process periodically sends hardware and process data. The default heartbeat period is 1 second. However, arcee can be initialized with a custom period", - "mlProfilingIntegration.initialization.customEndpointAdnSslChecks": "To use a custom endpoint and enable/disable SSL checks (enable self-signed SSL certificates support)", "mlProfilingIntegration.initialization.description": "To initialize the arcee collector use the init method with the following parameters:", - "mlProfilingIntegration.initialization.forCustomOptScaleDeployment": "For custom OptScale deployments", "mlProfilingIntegration.initialization.initCollectorUsingContextManager": "To initialize the collector using a context manager, use the following code snippet", "mlProfilingIntegration.initialization.initCollectorUsingContextManagerDescription": "This method automatically handles error catching and terminates arcee execution.", "mlProfilingIntegration.initialization.parameters.1.token": "token (str, required): profiling token",