From e262aea4bab0dffec821486a71cfc2df0769f2e1 Mon Sep 17 00:00:00 2001 From: Usmonbek Ravshanov Date: Thu, 13 Jul 2023 17:19:36 +0500 Subject: [PATCH 01/16] Bug Fix: integration title display problem fixed in security plugins --- static/js/security_app_create.js | 3 ++- static/js/security_code_create.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/static/js/security_app_create.js b/static/js/security_app_create.js index 6795f8f..03f2c60 100644 --- a/static/js/security_app_create.js +++ b/static/js/security_app_create.js @@ -161,7 +161,8 @@ const TestIntegrationItem = { }, methods: { getIntegrationTitle(integration) { - return integration.is_default ? `${integration.description} - default` : integration.description + integrationName = integration.hasOwnProperty('config')? integration.config.name : integration.description + return integration.is_default ? `${integrationName} - default` : integrationName }, clear_data() { this.is_selected = false diff --git a/static/js/security_code_create.js b/static/js/security_code_create.js index bbb96e3..109fa19 100644 --- a/static/js/security_code_create.js +++ b/static/js/security_code_create.js @@ -153,7 +153,8 @@ const TestIntegrationItem = { }, methods: { getIntegrationTitle(integration) { - return integration.is_default ? `${integration.description} - default` : integration.description + integrationName = integration.hasOwnProperty('config')? integration.config.name : integration.description + return integration.is_default ? `${integrationName} - default` : integrationName }, clear_data() { this.is_selected = false From f0f20c8d3339fd2bb1c18a50797ebdea11304598 Mon Sep 17 00:00:00 2001 From: Aspect13 Date: Thu, 27 Jul 2023 19:53:04 +0400 Subject: [PATCH 02/16] refactoring permissions and project steps --- events/__init__.py | 0 events/main.py | 37 +++++++++++++++++++++++++++++++++++++ module.py | 5 +++-- rpc/main.py | 39 +++++++++++++++++++-------------------- 4 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 events/__init__.py create mode 100644 events/main.py diff --git a/events/__init__.py b/events/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/events/main.py b/events/main.py new file mode 100644 index 0000000..ed5ae15 --- /dev/null +++ b/events/main.py @@ -0,0 +1,37 @@ +from sqlalchemy import Boolean + +from ..models.integration import IntegrationAdmin, IntegrationDefault +from ..models.pd.integration import SecretField + +from tools import rpc_tools, VaultClient, db + +from pylon.core.tools import web, log + + +def _usecret_field(integration_db, project_id): + settings = integration_db.settings + secret_access_key = SecretField.parse_obj(settings['secret_access_key']) + settings['secret_access_key'] = secret_access_key.unsecret(project_id=project_id) + return settings + + +class Event: + @web.event('project_created') + def create_default_s3_for_new_project(self, context, event, project: dict, **kwargs) -> None: + log.info('Creating default integration for project %s', project) + project_id = project['id'] + if integration_db := IntegrationAdmin.query.filter( + IntegrationAdmin.name == 's3_integration', + IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, + IntegrationAdmin.is_default == True, + ).one_or_none(): + with db.with_project_schema_session(project_id) as tenant_session: + default_integration = IntegrationDefault( + name=integration_db.name, + project_id=None, + integration_id=integration_db.id, + is_default=True, + section=integration_db.section + ) + tenant_session.add(default_integration) + tenant_session.commit() diff --git a/module.py b/module.py index 02004f8..65417c3 100644 --- a/module.py +++ b/module.py @@ -19,8 +19,8 @@ from pylon.core.tools import log # pylint: disable=E0611,E0401 from pylon.core.tools import module -from .models.integration import IntegrationAdmin # pylint: disable=E0611,E0401 -from .models.pd.integration import IntegrationBase +# from .models.integration import IntegrationAdmin # pylint: disable=E0611,E0401 +# from .models.pd.integration import IntegrationBase from .init_db import init_db @@ -46,6 +46,7 @@ def init(self): self.descriptor.init_blueprint() self.descriptor.init_api() self.descriptor.init_slots() + self.descriptor.init_events() theme.register_subsection( 'configuration', 'integrations', diff --git a/rpc/main.py b/rpc/main.py index 3728fd2..0c4f71a 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -132,7 +132,6 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ return IntegrationAdmin.query.filter( IntegrationAdmin.id == integration_id, ).one_or_none() - @web.rpc('security_test_create_integrations') @rpc_tools.wrap_exceptions(ValidationError) @@ -562,7 +561,7 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): return _usecret_field(integration_db, project_id) except Exception as e: log.warning(f'Cannot receive S3 settings for project {project_id}') - log.warning(e) + log.debug(e) @rpc('get_s3_admin_settings') def get_s3_admin_settings(self, integration_id=None): @@ -583,21 +582,21 @@ def get_s3_admin_settings(self, integration_id=None): return _usecret_field(integration_db, None) except Exception as e: log.warning(f'Cannot receive S3 settings in administration mode') - log.warning(e) - - @rpc('create_default_s3_for_new_project') - def create_default_s3_for_new_project(self, project_id): - if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.name == 's3_integration', - IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, - IntegrationAdmin.is_default == True, - ).one_or_none(): - with db.with_project_schema_session(project_id) as tenant_session: - default_integration = IntegrationDefault(name=integration_db.name, - project_id=None, - integration_id = integration_db.id, - is_default=True, - section=integration_db.section - ) - tenant_session.add(default_integration) - tenant_session.commit() + log.debug(e) + + # @rpc('create_default_s3_for_new_project') + # def create_default_s3_for_new_project(self, project_id): + # if integration_db := IntegrationAdmin.query.filter( + # IntegrationAdmin.name == 's3_integration', + # IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, + # IntegrationAdmin.is_default == True, + # ).one_or_none(): + # with db.with_project_schema_session(project_id) as tenant_session: + # default_integration = IntegrationDefault(name=integration_db.name, + # project_id=None, + # integration_id = integration_db.id, + # is_default=True, + # section=integration_db.section + # ) + # tenant_session.add(default_integration) + # tenant_session.commit() From 76b0f7b48bc8bc544f88a20af2decc5f099094ce Mon Sep 17 00:00:00 2001 From: Yury Miklashevich1 Date: Wed, 9 Aug 2023 11:16:45 +0300 Subject: [PATCH 03/16] Update for s3 client --- rpc/main.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rpc/main.py b/rpc/main.py index 0c4f71a..8f26b12 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -17,10 +17,12 @@ from pylon.core.tools import web -def _usecret_field(integration_db, project_id): +def _usecret_field(integration_db, project_id, is_local): settings = integration_db.settings secret_access_key = SecretField.parse_obj(settings['secret_access_key']) settings['secret_access_key'] = secret_access_key.unsecret(project_id=project_id) + settings['integration_id'] = integration_db.id + settings['is_local'] = is_local return settings @@ -532,14 +534,14 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): IntegrationProject.id == integration_id, IntegrationProject.name == integration_name ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=True) elif integration_id: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=False) # in case if integration_id is not provided - try to find default integration: else: with db.with_project_schema_session(project_id) as tenant_session: @@ -551,14 +553,14 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): IntegrationProject.id == default_integration.integration_id, IntegrationProject.name == integration_name ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=True) elif default_integration: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.id == default_integration.integration_id, IntegrationAdmin.name == integration_name, IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=False) except Exception as e: log.warning(f'Cannot receive S3 settings for project {project_id}') log.debug(e) @@ -572,14 +574,14 @@ def get_s3_admin_settings(self, integration_id=None): IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, ).one_or_none(): - return _usecret_field(integration_db, None) + return _usecret_field(integration_db, None, is_local=False) # in case if integration_id is not provided - try to find default integration: else: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.name == integration_name, IntegrationAdmin.is_default == True, ).one_or_none(): - return _usecret_field(integration_db, None) + return _usecret_field(integration_db, None, is_local=False) except Exception as e: log.warning(f'Cannot receive S3 settings in administration mode') log.debug(e) From fccc49ec60a91c81a6cdc0893ffbbde743070da2 Mon Sep 17 00:00:00 2001 From: Yury Miklashevich1 Date: Wed, 30 Aug 2023 01:49:58 +0300 Subject: [PATCH 04/16] added uid --- models/integration.py | 47 ++++++++++------------------------------ models/pd/integration.py | 23 +++++++------------- rpc/main.py | 33 ++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/models/integration.py b/models/integration.py index d8410e1..3daf82f 100644 --- a/models/integration.py +++ b/models/integration.py @@ -1,6 +1,7 @@ from pylon.core.tools import log from sqlalchemy import Integer, Column, String, Boolean, UniqueConstraint, Index from sqlalchemy.dialects.postgresql import JSON +from uuid import uuid4 from tools import db_tools, db, rpc_tools from ..models.pd.integration import IntegrationBase @@ -19,15 +20,15 @@ class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, ) id = Column(Integer, primary_key=True) name = Column(String(64), unique=False) - # project_id = Column(Integer, unique=False, nullable=True) - # mode = Column(String(64), unique=False, default='default') settings = Column(JSON, unique=False, default={}) is_default = Column(Boolean, default=False, nullable=False) section = Column(String(64), unique=False, nullable=False) - # description = Column(String(256), unique=False, nullable=True, default='Default integration') config = Column(JSON, unique=False, default={}) task_id = Column(String(256), unique=False, nullable=True) status = Column(String(256), unique=False, nullable=False, default='success') + # ALTER TABLE "Project-1"."integration" ADD COLUMN uid VARCHAR(128) + # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL + uid = Column(String(128), unique=True, nullable=False) def make_default(self): IntegrationAdmin.query.filter( @@ -45,6 +46,8 @@ def set_task_id(self, task_id: str): self.insert() def insert(self): + if not self.uid: + self.uid = str(uuid4()) if not IntegrationAdmin.query.filter( IntegrationAdmin.name == self.name, IntegrationAdmin.is_default == True, @@ -66,50 +69,24 @@ def process_secret_fields(self): class IntegrationProject(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): __tablename__ = "integration" - # __table_args__ = ( - # Index( - # 'ix_project_default_uc', # Index name - # 'project_id', 'name', # Columns which are part of the index - # unique=True, - # postgresql_where=Column('is_default') # The condition - # ) - # ) __table_args__ = {'schema': 'tenant'} id = Column(Integer, primary_key=True) name = Column(String(64), unique=False) project_id = Column(Integer, unique=False, nullable=True) - # mode = Column(String(64), unique=False, default='default') settings = Column(JSON, unique=False, default={}) is_default = Column(Boolean, default=False, nullable=False) section = Column(String(64), unique=False, nullable=False) - # description = Column(String(256), unique=False, nullable=True, default='Default integration') config = Column(JSON, unique=False, default={}) task_id = Column(String(256), unique=False, nullable=True) status = Column(String(256), unique=False, nullable=False, default='success') - - - # def make_default(self, session): - # default_integration = session.query(IntegrationProject).filter( - # IntegrationProject.project_id == self.project_id, - # IntegrationProject.name == self.name, - # IntegrationProject.is_default == True, - # IntegrationProject.id != self.id - # ).one_or_none() - # if default_integration: - # default_integration.is_default = False - # self.is_default = True - # # super().insert() - # session.commit() - - # def set_task_id(self, session, task_id: str): - # session.query(IntegrationProject).filter( - # IntegrationProject.id == self.id - # ).update({IntegrationProject.task_id: task_id}) - # # self.insert() - # session.commit() + # ALTER TABLE "Project-1"."integration" ADD COLUMN uid VARCHAR(128) + # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL + uid = Column(String(128), unique=True, nullable=False) def insert(self, session): + if not self.uid: + self.uid = str(uuid4()) session.add(self) session.commit() inherited_integration = IntegrationAdmin.query.filter( @@ -122,7 +99,6 @@ def insert(self, session): ).one_or_none() if not inherited_integration and not default_integration: self.rpc.call.integrations_make_default_integration(self, self.project_id) - # super().insert(session) self.process_secret_fields(session) self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) @@ -133,7 +109,6 @@ def process_secret_fields(self, session): session.query(IntegrationProject).filter( IntegrationProject.id == self.id ).update({IntegrationProject.settings: settings}) - # super().insert() session.commit() diff --git a/models/pd/integration.py b/models/pd/integration.py index f0c3175..8c4f95c 100644 --- a/models/pd/integration.py +++ b/models/pd/integration.py @@ -1,4 +1,5 @@ from typing import Optional, Union +from uuid import uuid4 from pydantic import BaseModel, validator, constr from pylon.core.tools import log @@ -18,13 +19,19 @@ class IntegrationBase(BaseModel): config: dict task_id: Optional[str] status: Optional[str] = 'success' - # mode: str + uid: str class Config: orm_mode = True class IntegrationPD(IntegrationBase): + @validator('uid', pre=True, always=True) + def set_uid(cls, value: Optional[str]): + if not value: + return str(uuid4()) + return value + @validator("settings") def validate_settings(cls, value, values): integration = rpc_tools.RpcMixin().rpc.call.integrations_get_by_name( @@ -44,11 +51,6 @@ def validate_section(cls, value, values): return rpc_tools.RpcMixin().rpc.call.integrations_register_section(name=value) return section - # @validator("config") - # def validate_config(cls, value, values): - # assert value.get('name'), 'ensure this value has at least 1 characters' - # return value - @validator("config") def validate_description(cls, value, values): if not value.get('name'): @@ -57,15 +59,6 @@ def validate_description(cls, value, values): return value -# class IntegrationProjectPD(IntegrationPD): -# pass - # @validator("is_default") - # def validate_is_default(cls, value, values): - # if rpc_tools.RpcMixin().rpc.call.integrations_is_default(values['project_id'], values): - # return True - # return False - - class IntegrationDefaultPD(BaseModel): id: int name: str diff --git a/rpc/main.py b/rpc/main.py index 8f26b12..ab8208f 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -126,6 +126,13 @@ def section_list(self) -> list: @rpc('get_by_id') def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[IntegrationProject]: + """ + Get integration by id. Works properly if you know: inherited this integration or not. + :param project_id: id of project, where integration was created. If None - integration + from administration mode + :param integration_id: id of integration + :return: integration ORM object or None + """ if project_id is not None: with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationProject).filter( @@ -134,6 +141,32 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ return IntegrationAdmin.query.filter( IntegrationAdmin.id == integration_id, ).one_or_none() + + @rpc('get_by_uid') + def get_by_uid(self, integration_uid: int, project_id: Optional[int] = None) -> Optional[IntegrationProject]: + """ + Get integration by unique id. You can specify current project_id but not necessary. + :param integration_uid: uid of integration + :param project_id: id of current project + :return: integration ORM object or None + """ + if project_id is not None: + with db.with_project_schema_session(project_id) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration + if integration := IntegrationAdmin.query.filter( + IntegrationAdmin.uid == integration_uid, + ).one_or_none(): + return integration + projects = self.context.rpc_manager.call.project_list() + for project in projects: + with db.with_project_schema_session(project['id']) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration @web.rpc('security_test_create_integrations') @rpc_tools.wrap_exceptions(ValidationError) From 2693689404f2d6ef709ecb5021a2fde90592e8c9 Mon Sep 17 00:00:00 2001 From: Yury Miklashevich1 Date: Tue, 5 Sep 2023 18:30:03 +0300 Subject: [PATCH 05/16] fix for integration_uid --- rpc/main.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/rpc/main.py b/rpc/main.py index ab8208f..5e6cb6a 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -143,11 +143,16 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ ).one_or_none() @rpc('get_by_uid') - def get_by_uid(self, integration_uid: int, project_id: Optional[int] = None) -> Optional[IntegrationProject]: + def get_by_uid( + self, integration_uid: int, + project_id: Optional[int] = None, + check_all_projects: bool = True + ) -> Optional[IntegrationProject]: """ Get integration by unique id. You can specify current project_id but not necessary. :param integration_uid: uid of integration :param project_id: id of current project + :param check_all_projects: True - if we want to search in all projects :return: integration ORM object or None """ if project_id is not None: @@ -160,13 +165,14 @@ def get_by_uid(self, integration_uid: int, project_id: Optional[int] = None) -> IntegrationAdmin.uid == integration_uid, ).one_or_none(): return integration - projects = self.context.rpc_manager.call.project_list() - for project in projects: - with db.with_project_schema_session(project['id']) as tenant_session: - if integration := tenant_session.query(IntegrationProject).filter( - IntegrationProject.uid == integration_uid, - ).one_or_none(): - return integration + if check_all_projects: + projects = self.context.rpc_manager.call.project_list() + for project in projects: + with db.with_project_schema_session(project['id']) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration @web.rpc('security_test_create_integrations') @rpc_tools.wrap_exceptions(ValidationError) From b02d4c4f4b3e2f257ea93beed9f82749074c99d2 Mon Sep 17 00:00:00 2001 From: Yury Miklashevich1 Date: Tue, 5 Sep 2023 21:50:33 +0300 Subject: [PATCH 06/16] check up integration_uid type --- rpc/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rpc/main.py b/rpc/main.py index 5e6cb6a..fbfa1b1 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -144,7 +144,7 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ @rpc('get_by_uid') def get_by_uid( - self, integration_uid: int, + self, integration_uid: str, project_id: Optional[int] = None, check_all_projects: bool = True ) -> Optional[IntegrationProject]: @@ -154,7 +154,9 @@ def get_by_uid( :param project_id: id of current project :param check_all_projects: True - if we want to search in all projects :return: integration ORM object or None - """ + """ + if not isinstance(integration_uid, str): + integration_uid = str(integration_uid) if project_id is not None: with db.with_project_schema_session(project_id) as tenant_session: if integration := tenant_session.query(IntegrationProject).filter( From 5ff7b35b5066bb406822f01981a5859e6b417d3b Mon Sep 17 00:00:00 2001 From: Aspect13 Date: Tue, 24 Oct 2023 11:20:25 +0400 Subject: [PATCH 07/16] integration details by uid api --- api/v1/integration.py | 52 ++++++++++++++++++++++++++++++++++++++++--- rpc/main.py | 3 +-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/api/v1/integration.py b/api/v1/integration.py index db39dae..7fda338 100644 --- a/api/v1/integration.py +++ b/api/v1/integration.py @@ -9,6 +9,31 @@ class ProjectAPI(api_tools.APIModeHandler): + @auth.decorators.check_api({ + "permissions": ["configuration.integrations.integration.details"], + "recommended_roles": { + "administration": {"admin": True, "viewer": True, "editor": True}, + "default": {"admin": True, "viewer": True, "editor": True}, + "developer": {"admin": False, "viewer": False, "editor": False}, + }}) + def get(self, project_id: int, integration_uid: int, **kwargs): + integration = self.module.get_by_uid( + integration_uid=integration_uid, + project_id=project_id, + check_all_projects=False + ) + if not integration: + integration = self.module.get_by_uid( + integration_id=integration_uid, + ) + if not integration: + return None, 404 + return IntegrationPD.from_orm(integration).dict(), 200 + # try: + # settings = integration.settings_model.parse_obj(request.json) + # except ValidationError as e: + # return e.errors(), 400 + @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.create"], "recommended_roles": { @@ -43,7 +68,7 @@ def post(self, integration_name: str): try: return IntegrationPD.from_orm(db_integration).dict(), 200 except ValidationError as e: - return e.errors(), 400 + return e.errors(), 400 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -108,7 +133,7 @@ def patch(self, project_id: int, integration_id: int): "administration": {"admin": True, "viewer": False, "editor": False}, "default": {"admin": True, "viewer": False, "editor": False}, "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + }}) def delete(self, project_id: int, integration_id: int): with db.with_project_schema_session(project_id) as tenant_session: db_integration = tenant_session.query(IntegrationProject).filter( @@ -121,6 +146,21 @@ def delete(self, project_id: int, integration_id: int): class AdminAPI(api_tools.APIModeHandler): + @auth.decorators.check_api({ + "permissions": ["configuration.integrations.integration.details"], + "recommended_roles": { + "administration": {"admin": True, "viewer": True, "editor": True}, + "default": {"admin": True, "viewer": True, "editor": True}, + "developer": {"admin": False, "viewer": False, "editor": False}, + }}) + def get(self, project_id: int, integration_uid: int, **kwargs): + integration = self.module.get_by_uid( + integration_uid=integration_uid, + ) + if not integration: + return None, 404 + return IntegrationPD.from_orm(integration).dict(), 200 + @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.create"], "recommended_roles": { @@ -197,20 +237,26 @@ def patch(self, integration_id: int, **kwargs): "administration": {"admin": True, "viewer": False, "editor": False}, "default": {"admin": True, "viewer": False, "editor": False}, "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + }}) def delete(self, integration_id: int, **kwargs): IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).delete() IntegrationAdmin.commit() return integration_id, 204 + class API(api_tools.APIBase): url_params = [ '', '/', + '', '/', + '/', '//', + + '/', + '//', ] mode_handlers = { diff --git a/rpc/main.py b/rpc/main.py index fbfa1b1..8134a9c 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -155,8 +155,7 @@ def get_by_uid( :param check_all_projects: True - if we want to search in all projects :return: integration ORM object or None """ - if not isinstance(integration_uid, str): - integration_uid = str(integration_uid) + integration_uid = str(integration_uid) if project_id is not None: with db.with_project_schema_session(project_id) as tenant_session: if integration := tenant_session.query(IntegrationProject).filter( From 961774f10e9bb60855ff22929af639f4976f7a73 Mon Sep 17 00:00:00 2001 From: Usmonbek Ravshanov Date: Wed, 31 Jan 2024 12:39:40 +0500 Subject: [PATCH 08/16] project_id is passed to check_connection project_id is passed to check_connection in check_settings endpoint --- api/v1/check_settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/v1/check_settings.py b/api/v1/check_settings.py index ee8892b..e559c6f 100644 --- a/api/v1/check_settings.py +++ b/api/v1/check_settings.py @@ -28,15 +28,18 @@ class API(api_tools.APIBase): ]) def post(self, integration_name: str, **kwargs): integration = self.module.get_by_name(integration_name) + payload = request.json if not integration: return {'error': 'integration not found'}, 404 try: - settings = integration.settings_model.parse_obj(request.json) + settings = integration.settings_model.parse_obj(payload) except ValidationError as e: # return e.json(), 400 return e.errors(), 400 - check_connection_response = settings.check_connection() + project_id = payload.get('project_id') + project_id = int(project_id) if project_id else project_id + check_connection_response = settings.check_connection(project_id) if not request.json.get('save_action'): if check_connection_response is True: return 'OK', 200 From 7d46e29a2a3ffc2f2d44b35ff90a1ed0a747bd98 Mon Sep 17 00:00:00 2001 From: Ivan Krakhmaliuk Date: Wed, 31 Jan 2024 15:04:02 +0200 Subject: [PATCH 09/16] specify the use of project_id from request.json in auth decorator --- api/v1/check_settings.py | 12 ++++++++---- api/v1/integration.py | 36 ++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/api/v1/check_settings.py b/api/v1/check_settings.py index e559c6f..389f5a1 100644 --- a/api/v1/check_settings.py +++ b/api/v1/check_settings.py @@ -23,9 +23,13 @@ class API(api_tools.APIBase): 'administration': AdminAPI, } - @auth.decorators.check_api(["configuration.integrations.integrations.create", - "configuration.integrations.integrations.edit" - ]) + @auth.decorators.check_api( + [ + "configuration.integrations.integrations.create", + "configuration.integrations.integrations.edit" + ], + project_id_in_request_json=True + ) def post(self, integration_name: str, **kwargs): integration = self.module.get_by_name(integration_name) payload = request.json @@ -38,7 +42,7 @@ def post(self, integration_name: str, **kwargs): return e.errors(), 400 project_id = payload.get('project_id') - project_id = int(project_id) if project_id else project_id + project_id = int(project_id) if project_id else project_id check_connection_response = settings.check_connection(project_id) if not request.json.get('save_action'): if check_connection_response is True: diff --git a/api/v1/integration.py b/api/v1/integration.py index 7fda338..f89255f 100644 --- a/api/v1/integration.py +++ b/api/v1/integration.py @@ -34,13 +34,17 @@ def get(self, project_id: int, integration_uid: int, **kwargs): # except ValidationError as e: # return e.errors(), 400 - @auth.decorators.check_api({ - "permissions": ["configuration.integrations.integrations.create"], - "recommended_roles": { - "administration": {"admin": True, "viewer": False, "editor": True}, - "default": {"admin": True, "viewer": False, "editor": True}, - "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + @auth.decorators.check_api( + { + "permissions": ["configuration.integrations.integrations.create"], + "recommended_roles": { + "administration": {"admin": True, "viewer": False, "editor": True}, + "default": {"admin": True, "viewer": False, "editor": True}, + "developer": {"admin": False, "viewer": False, "editor": False}, + } + }, + project_id_in_request_json=True + ) def post(self, integration_name: str): project_id = request.json.get('project_id') if not project_id: @@ -70,13 +74,17 @@ def post(self, integration_name: str): except ValidationError as e: return e.errors(), 400 - @auth.decorators.check_api({ - "permissions": ["configuration.integrations.integrations.edit"], - "recommended_roles": { - "administration": {"admin": True, "viewer": False, "editor": True}, - "default": {"admin": True, "viewer": False, "editor": True}, - "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + @auth.decorators.check_api( + { + "permissions": ["configuration.integrations.integrations.edit"], + "recommended_roles": { + "administration": {"admin": True, "viewer": False, "editor": True}, + "default": {"admin": True, "viewer": False, "editor": True}, + "developer": {"admin": False, "viewer": False, "editor": False}, + } + }, + project_id_in_request_json=True + ) def put(self, integration_id: int): project_id = request.json.get('project_id') if not project_id: From 75b406b4db209adadbb39c5ed6566f880bca8626 Mon Sep 17 00:00:00 2001 From: Daniil_Motsnyi Date: Fri, 9 Feb 2024 13:47:10 +0200 Subject: [PATCH 10/16] Adding pagination and sorting for the integrations --- api/v1/integrations.py | 31 +++++++++++++++++++++++++++++++ rpc/main.py | 14 ++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/api/v1/integrations.py b/api/v1/integrations.py index 6d43d36..e75ed21 100644 --- a/api/v1/integrations.py +++ b/api/v1/integrations.py @@ -48,6 +48,36 @@ def get(self, **kwargs): ], 200 +class ModelsAPI(api_tools.APIModeHandler): + AI_SECTION: str = 'ai' + + @auth.decorators.check_api({ + "permissions": ["configuration.integrations.integrations.view"], + "recommended_roles": { + "administration": {"admin": True, "viewer": True, "editor": True}, + "default": {"admin": True, "viewer": True, "editor": True}, + "developer": {"admin": True, "viewer": False, "editor": False}, + }}) + def get(self, project_id: int): + sort_order, sort_field = 'asc', 'name' + page, per_page = None, 10 + + if request.args.get('sort_order'): + sort_order = request.args.get('sort_order') + if request.args.get('sort_field'): + sort_field = request.args.get('sort_field') + if request.args.get('page'): + page = int(request.args.get('page')) + if request.args.get('size'): + per_page = int(request.args.get('size')) + + return [ + i.dict() for i in self.module.get_sorted_paginated_integrations_by_section( + self.AI_SECTION, project_id, sort_order, sort_field, page, per_page + ) + ], 200 + + class API(api_tools.APIBase): url_params = [ '', @@ -59,4 +89,5 @@ class API(api_tools.APIBase): mode_handlers = { 'default': ProjectAPI, 'administration': AdminAPI, + 'prompt_lib': ModelsAPI, } diff --git a/rpc/main.py b/rpc/main.py index 8134a9c..66969e3 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -472,6 +472,20 @@ def get_all_integrations_by_section(self, project_id: int, section_name: str) -> results_admin = self.get_administration_integrations_by_section(section_name, True) return self.process_default_integrations(project_id, results_project + results_admin) + @rpc('get_sorted_paginated_integrations_by_section') + def get_sorted_paginated_integrations_by_section(self, section_name: str, project_id: int, sort_order: str, + sort_field: str, page: int, per_page: int): + results_project = self.get_project_integrations_by_section(project_id, section_name) + results_admin = self.get_administration_integrations_by_section(section_name, True) + results = parse_obj_as(List[IntegrationPD], results_project + results_admin) + descending = sort_order.lower() == 'desc' + sorted_list = sorted(results, key=lambda x: getattr(x, sort_field), reverse=descending) + if page is None or page < 1: + return sorted_list + offset = (page - 1) * per_page + limit = offset + per_page + return sorted_list[offset:limit] + @rpc('update_attrs') def update_attrs(self, integration_id: int, From 5ea734d61a707cb198eea66a0fd6796163c4005e Mon Sep 17 00:00:00 2001 From: Daniil Motsnyi Date: Fri, 9 Feb 2024 15:27:18 +0200 Subject: [PATCH 11/16] params fixed --- api/v1/integrations.py | 22 +++++++--------------- rpc/main.py | 8 ++------ 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/api/v1/integrations.py b/api/v1/integrations.py index e75ed21..481a55d 100644 --- a/api/v1/integrations.py +++ b/api/v1/integrations.py @@ -48,7 +48,7 @@ def get(self, **kwargs): ], 200 -class ModelsAPI(api_tools.APIModeHandler): +class PromptLibAPI(api_tools.APIModeHandler): AI_SECTION: str = 'ai' @auth.decorators.check_api({ @@ -59,21 +59,13 @@ class ModelsAPI(api_tools.APIModeHandler): "developer": {"admin": True, "viewer": False, "editor": False}, }}) def get(self, project_id: int): - sort_order, sort_field = 'asc', 'name' - page, per_page = None, 10 - - if request.args.get('sort_order'): - sort_order = request.args.get('sort_order') - if request.args.get('sort_field'): - sort_field = request.args.get('sort_field') - if request.args.get('page'): - page = int(request.args.get('page')) - if request.args.get('size'): - per_page = int(request.args.get('size')) - + sort_order = request.args.get('sort_order', 'asc') + sort_by = request.args.get('sort_by', 'name') + offset = int(request.args.get('offset', 0)) + limit = int(request.args.get('limit', 10_000)) return [ i.dict() for i in self.module.get_sorted_paginated_integrations_by_section( - self.AI_SECTION, project_id, sort_order, sort_field, page, per_page + self.AI_SECTION, project_id, sort_order, sort_by, offset, limit ) ], 200 @@ -89,5 +81,5 @@ class API(api_tools.APIBase): mode_handlers = { 'default': ProjectAPI, 'administration': AdminAPI, - 'prompt_lib': ModelsAPI, + 'prompt_lib': PromptLibAPI, } diff --git a/rpc/main.py b/rpc/main.py index 66969e3..015eb5a 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -474,16 +474,12 @@ def get_all_integrations_by_section(self, project_id: int, section_name: str) -> @rpc('get_sorted_paginated_integrations_by_section') def get_sorted_paginated_integrations_by_section(self, section_name: str, project_id: int, sort_order: str, - sort_field: str, page: int, per_page: int): + sort_by: str, offset: int, limit: int): results_project = self.get_project_integrations_by_section(project_id, section_name) results_admin = self.get_administration_integrations_by_section(section_name, True) results = parse_obj_as(List[IntegrationPD], results_project + results_admin) descending = sort_order.lower() == 'desc' - sorted_list = sorted(results, key=lambda x: getattr(x, sort_field), reverse=descending) - if page is None or page < 1: - return sorted_list - offset = (page - 1) * per_page - limit = offset + per_page + sorted_list = sorted(results, key=lambda x: getattr(x, sort_by), reverse=descending) return sorted_list[offset:limit] @rpc('update_attrs') From 60cdfdae92bf5e53bd6f9cfaa45780e0ce74c4d0 Mon Sep 17 00:00:00 2001 From: Daniil Motsnyi Date: Thu, 22 Feb 2024 13:55:31 +0200 Subject: [PATCH 12/16] Available integrations list --- api/v1/available.py | 31 +++++++++++++++++++++++++++++++ rpc/main.py | 4 ++++ 2 files changed, 35 insertions(+) create mode 100644 api/v1/available.py diff --git a/api/v1/available.py b/api/v1/available.py new file mode 100644 index 0000000..29d1d61 --- /dev/null +++ b/api/v1/available.py @@ -0,0 +1,31 @@ +from flask import request + +from tools import auth, api_tools + + +class ProjectAPI(api_tools.APIModeHandler): + ... + + +class AdminAPI(api_tools.APIModeHandler): + ... + + +class API(api_tools.APIBase): + url_params = [ + '', + '/', + '', + '/' + ] + + mode_handlers = { + 'default': ProjectAPI, + 'administration': AdminAPI, + } + + def get(self, **kwargs): + section = request.args.get('section') + if section: + return self.module.list_integrations_by_section(section), 200 + return list(self.module.list_integrations()), 200 diff --git a/rpc/main.py b/rpc/main.py index 015eb5a..8d0db30 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -44,6 +44,10 @@ def get_by_name(self, integration_name: str) -> Optional[RegistrationForm]: def list_integrations(self) -> dict: return self.integrations + @rpc('list_integrations_by_section') + def list_integrations_by_section(self, section: str) -> list: + return [k for k, v in self.integrations.items() if v.section == section] + @rpc('get_project_integrations') def get_project_integrations(self, project_id: int, group_by_section: bool = True) -> dict: with db.with_project_schema_session(project_id) as tenant_session: From bd98b24a97986879097ee7f87e32431bd3c512d2 Mon Sep 17 00:00:00 2001 From: Aspect13 Date: Thu, 14 Mar 2024 19:54:02 +0400 Subject: [PATCH 13/16] integrations - set project id to found integration refactoring prompt_lib for stream --- rpc/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rpc/main.py b/rpc/main.py index 8d0db30..dfdc654 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -165,6 +165,7 @@ def get_by_uid( if integration := tenant_session.query(IntegrationProject).filter( IntegrationProject.uid == integration_uid, ).one_or_none(): + integration.project_id = project_id return integration if integration := IntegrationAdmin.query.filter( IntegrationAdmin.uid == integration_uid, @@ -177,6 +178,7 @@ def get_by_uid( if integration := tenant_session.query(IntegrationProject).filter( IntegrationProject.uid == integration_uid, ).one_or_none(): + integration.project_id = project['id'] return integration @web.rpc('security_test_create_integrations') From 2c053bcf9b648c038d6d3c300295950ec81167db Mon Sep 17 00:00:00 2001 From: Aspect13 Date: Wed, 17 Apr 2024 14:54:31 +0400 Subject: [PATCH 14/16] integrations session management fix --- api/v1/integration.py | 78 ++++++++++++++++++++++++------------------- models/integration.py | 40 +++++++++++----------- rpc/main.py | 57 ++++++++++++++++--------------- 3 files changed, 93 insertions(+), 82 deletions(-) diff --git a/api/v1/integration.py b/api/v1/integration.py index f89255f..0a90d63 100644 --- a/api/v1/integration.py +++ b/api/v1/integration.py @@ -1,7 +1,7 @@ from pylon.core.tools import log from flask import request -from pydantic import ValidationError, parse_obj_as +from pydantic import ValidationError from tools import api_tools, auth, db from ...models.integration import IntegrationProject, IntegrationAdmin @@ -126,13 +126,14 @@ def patch(self, project_id: int, integration_id: int): return {'error': 'integration not found'}, 404 self.module.make_default_integration(db_integration, project_id) else: - db_integration = IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - db_integration.project_id = None - self.module.make_default_integration(db_integration, project_id) + with db.get_session() as session: + db_integration = session.query(IntegrationAdmin).where( + IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + db_integration.project_id = None + self.module.make_default_integration(db_integration, project_id) return {'msg': 'integration set as default'}, 200 @auth.decorators.check_api({ @@ -194,10 +195,11 @@ def post(self, integration_name: str, **kwargs): config=request.json.get('config'), status=request.json.get('status', 'success'), ) - db_integration.insert() - if request.json.get('is_default'): - db_integration.make_default() - return IntegrationPD.from_orm(db_integration).dict(), 200 + with db.get_session() as session: + db_integration.insert(session) + if request.json.get('is_default'): + db_integration.make_default(session) + return IntegrationPD.from_orm(db_integration).dict(), 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -207,22 +209,24 @@ def post(self, integration_name: str, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def put(self, integration_id: int, **kwargs): - db_integration = IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - try: - settings = integration.settings_model.parse_obj(request.json) - except ValidationError as e: - return e.errors(), 400 + with db.get_session() as session: + db_integration = session.query(IntegrationAdmin).where(IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + try: + settings = integration.settings_model.parse_obj(request.json) + except ValidationError as e: + return e.errors(), 400 - if request.json.get('is_default'): - db_integration.make_default() + db_integration.settings = settings.dict() + db_integration.config = request.json.get('config') + db_integration.insert(session=session) - db_integration.settings = settings.dict() - db_integration.config = request.json.get('config') - db_integration.insert() - return IntegrationPD.from_orm(db_integration).dict(), 200 + if request.json.get('is_default'): + db_integration.make_default(session=session) + session.commit() + return IntegrationPD.from_orm(db_integration).dict(), 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -232,12 +236,15 @@ def put(self, integration_id: int, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def patch(self, integration_id: int, **kwargs): - db_integration = IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - db_integration.make_default() - return {'msg': 'integration set as default'}, 200 + with db.get_session() as session: + db_integration: IntegrationAdmin = session.query(IntegrationAdmin).where( + IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + db_integration.make_default(session=session) + session.commit() + return {'msg': 'integration set as default'}, 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.delete"], @@ -247,9 +254,10 @@ def patch(self, integration_id: int, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def delete(self, integration_id: int, **kwargs): - IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).delete() - IntegrationAdmin.commit() - return integration_id, 204 + with db.get_session() as session: + del_id = session.query(IntegrationAdmin.id).where(IntegrationAdmin.id == integration_id).delete() + session.commit() + return del_id, 204 class API(api_tools.APIBase): diff --git a/models/integration.py b/models/integration.py index 3daf82f..6a49a45 100644 --- a/models/integration.py +++ b/models/integration.py @@ -7,7 +7,6 @@ from ..models.pd.integration import IntegrationBase - class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): __tablename__ = "integration" __table_args__ = ( @@ -30,41 +29,42 @@ class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL uid = Column(String(128), unique=True, nullable=False) - def make_default(self): - IntegrationAdmin.query.filter( + def make_default(self, session): + session.query(IntegrationAdmin).where( IntegrationAdmin.name == self.name, IntegrationAdmin.is_default == True, IntegrationAdmin.id != self.id ).update({IntegrationAdmin.is_default: False}) self.is_default = True - super().insert() + # session.add(self) + # session.commit() def set_task_id(self, task_id: str): - IntegrationAdmin.query.filter( - IntegrationAdmin.id == self.id - ).update({IntegrationAdmin.task_id: task_id}) - self.insert() + with db.get_session() as session: + session.query(IntegrationAdmin).where( + IntegrationAdmin.id == self.id + ).update({IntegrationAdmin.task_id: task_id}) + session.commit() - def insert(self): + def insert(self, session): if not self.uid: self.uid = str(uuid4()) - if not IntegrationAdmin.query.filter( - IntegrationAdmin.name == self.name, - IntegrationAdmin.is_default == True, - ).one_or_none(): + if not session.query(IntegrationAdmin).filter( + IntegrationAdmin.name == self.name, + IntegrationAdmin.is_default == True, + ).first(): self.is_default = True - super().insert() - self.process_secret_fields() - self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) - - def process_secret_fields(self): + session.add(self) + # session.commit() settings: dict = self.rpc.call.integrations_process_secrets( integration_data=IntegrationBase.from_orm(self).dict(), ) - IntegrationAdmin.query.filter( + session.query(IntegrationAdmin).where( IntegrationAdmin.id == self.id ).update({IntegrationAdmin.settings: settings}) - super().insert() + # session.commit() + self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) + class IntegrationProject(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): diff --git a/rpc/main.py b/rpc/main.py index dfdc654..cf95cb4 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -141,10 +141,11 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationProject).filter( IntegrationProject.id == integration_id, - ).one_or_none() - return IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id, - ).one_or_none() + ).first() + with db.get_session() as session: + return session.query(IntegrationAdmin).where( + IntegrationAdmin.id == integration_id, + ).first() @rpc('get_by_uid') def get_by_uid( @@ -161,23 +162,24 @@ def get_by_uid( """ integration_uid = str(integration_uid) if project_id is not None: - with db.with_project_schema_session(project_id) as tenant_session: + with db.get_session(project_id) as tenant_session: if integration := tenant_session.query(IntegrationProject).filter( IntegrationProject.uid == integration_uid, ).one_or_none(): integration.project_id = project_id return integration - if integration := IntegrationAdmin.query.filter( - IntegrationAdmin.uid == integration_uid, - ).one_or_none(): - return integration + with db.get_session() as session: + if integration := session.query(IntegrationAdmin).where( + IntegrationAdmin.uid == integration_uid, + ).first(): + return integration if check_all_projects: projects = self.context.rpc_manager.call.project_list() for project in projects: - with db.with_project_schema_session(project['id']) as tenant_session: - if integration := tenant_session.query(IntegrationProject).filter( + with db.get_session(project['id']) as tenant_session: + if integration := tenant_session.query(IntegrationProject).where( IntegrationProject.uid == integration_uid, - ).one_or_none(): + ).first(): integration.project_id = project['id'] return integration @@ -345,23 +347,24 @@ def get_cloud_integrations(self, project_id: int) -> list: return cloud_regions @rpc('get_administration_integrations') - def get_administration_integrations(self, group_by_section: bool = True) -> dict: - results = IntegrationAdmin.query.filter( - IntegrationAdmin.name.in_(self.integrations.keys()) - ).group_by( - IntegrationAdmin.section, - IntegrationAdmin.id - ).order_by( - asc(IntegrationAdmin.section), - desc(IntegrationAdmin.is_default), - asc(IntegrationAdmin.name), - desc(IntegrationAdmin.id) - ).all() + def get_administration_integrations(self, group_by_section: bool = True) -> dict | List[IntegrationPD]: + with db.get_session() as session: + results = session.query(IntegrationAdmin).where( + IntegrationAdmin.name.in_(self.integrations.keys()) + ).group_by( + IntegrationAdmin.section, + IntegrationAdmin.id + ).order_by( + asc(IntegrationAdmin.section), + desc(IntegrationAdmin.is_default), + asc(IntegrationAdmin.name), + desc(IntegrationAdmin.id) + ).all() - results = parse_obj_as(List[IntegrationPD], results) + results = parse_obj_as(List[IntegrationPD], results) - if not group_by_section: - return results + if not group_by_section: + return results def reducer(accum: dict, new_value: IntegrationPD) -> dict: accum[new_value.section.name].append(new_value) From 81d3d8e27c5fb30f73f05802771eacc2ef7045c4 Mon Sep 17 00:00:00 2001 From: Aspect13 Date: Thu, 18 Apr 2024 17:15:42 +0400 Subject: [PATCH 15/16] integration create with id fix --- models/integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/integration.py b/models/integration.py index 6a49a45..86426bb 100644 --- a/models/integration.py +++ b/models/integration.py @@ -55,7 +55,8 @@ def insert(self, session): ).first(): self.is_default = True session.add(self) - # session.commit() + session.commit() + session.refresh(self) settings: dict = self.rpc.call.integrations_process_secrets( integration_data=IntegrationBase.from_orm(self).dict(), ) From b961a2fbc812327985d49f37bf90f59b1caf7b67 Mon Sep 17 00:00:00 2001 From: Vitalii_Kondratskyi Date: Wed, 12 Jun 2024 19:41:46 +0300 Subject: [PATCH 16/16] feat(DARK MODE): added dark mode; --- api/v1/integration.py | 154 +++++++++++++----------------------------- models/integration.py | 43 ++++++------ rpc/main.py | 138 +++++++++++++++---------------------- 3 files changed, 123 insertions(+), 212 deletions(-) diff --git a/api/v1/integration.py b/api/v1/integration.py index 0a90d63..8da2ea2 100644 --- a/api/v1/integration.py +++ b/api/v1/integration.py @@ -1,7 +1,7 @@ from pylon.core.tools import log from flask import request -from pydantic import ValidationError +from pydantic import ValidationError, parse_obj_as from tools import api_tools, auth, db from ...models.integration import IntegrationProject, IntegrationAdmin @@ -10,41 +10,12 @@ class ProjectAPI(api_tools.APIModeHandler): @auth.decorators.check_api({ - "permissions": ["configuration.integrations.integration.details"], + "permissions": ["configuration.integrations.integrations.create"], "recommended_roles": { - "administration": {"admin": True, "viewer": True, "editor": True}, - "default": {"admin": True, "viewer": True, "editor": True}, + "administration": {"admin": True, "viewer": False, "editor": True}, + "default": {"admin": True, "viewer": False, "editor": True}, "developer": {"admin": False, "viewer": False, "editor": False}, }}) - def get(self, project_id: int, integration_uid: int, **kwargs): - integration = self.module.get_by_uid( - integration_uid=integration_uid, - project_id=project_id, - check_all_projects=False - ) - if not integration: - integration = self.module.get_by_uid( - integration_id=integration_uid, - ) - if not integration: - return None, 404 - return IntegrationPD.from_orm(integration).dict(), 200 - # try: - # settings = integration.settings_model.parse_obj(request.json) - # except ValidationError as e: - # return e.errors(), 400 - - @auth.decorators.check_api( - { - "permissions": ["configuration.integrations.integrations.create"], - "recommended_roles": { - "administration": {"admin": True, "viewer": False, "editor": True}, - "default": {"admin": True, "viewer": False, "editor": True}, - "developer": {"admin": False, "viewer": False, "editor": False}, - } - }, - project_id_in_request_json=True - ) def post(self, integration_name: str): project_id = request.json.get('project_id') if not project_id: @@ -74,17 +45,13 @@ def post(self, integration_name: str): except ValidationError as e: return e.errors(), 400 - @auth.decorators.check_api( - { - "permissions": ["configuration.integrations.integrations.edit"], - "recommended_roles": { - "administration": {"admin": True, "viewer": False, "editor": True}, - "default": {"admin": True, "viewer": False, "editor": True}, - "developer": {"admin": False, "viewer": False, "editor": False}, - } - }, - project_id_in_request_json=True - ) + @auth.decorators.check_api({ + "permissions": ["configuration.integrations.integrations.edit"], + "recommended_roles": { + "administration": {"admin": True, "viewer": False, "editor": True}, + "default": {"admin": True, "viewer": False, "editor": True}, + "developer": {"admin": False, "viewer": False, "editor": False}, + }}) def put(self, integration_id: int): project_id = request.json.get('project_id') if not project_id: @@ -126,14 +93,13 @@ def patch(self, project_id: int, integration_id: int): return {'error': 'integration not found'}, 404 self.module.make_default_integration(db_integration, project_id) else: - with db.get_session() as session: - db_integration = session.query(IntegrationAdmin).where( - IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - db_integration.project_id = None - self.module.make_default_integration(db_integration, project_id) + db_integration = IntegrationAdmin.query.filter( + IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + db_integration.project_id = None + self.module.make_default_integration(db_integration, project_id) return {'msg': 'integration set as default'}, 200 @auth.decorators.check_api({ @@ -155,21 +121,6 @@ def delete(self, project_id: int, integration_id: int): class AdminAPI(api_tools.APIModeHandler): - @auth.decorators.check_api({ - "permissions": ["configuration.integrations.integration.details"], - "recommended_roles": { - "administration": {"admin": True, "viewer": True, "editor": True}, - "default": {"admin": True, "viewer": True, "editor": True}, - "developer": {"admin": False, "viewer": False, "editor": False}, - }}) - def get(self, project_id: int, integration_uid: int, **kwargs): - integration = self.module.get_by_uid( - integration_uid=integration_uid, - ) - if not integration: - return None, 404 - return IntegrationPD.from_orm(integration).dict(), 200 - @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.create"], "recommended_roles": { @@ -195,11 +146,10 @@ def post(self, integration_name: str, **kwargs): config=request.json.get('config'), status=request.json.get('status', 'success'), ) - with db.get_session() as session: - db_integration.insert(session) - if request.json.get('is_default'): - db_integration.make_default(session) - return IntegrationPD.from_orm(db_integration).dict(), 200 + db_integration.insert() + if request.json.get('is_default'): + db_integration.make_default() + return IntegrationPD.from_orm(db_integration).dict(), 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -209,24 +159,22 @@ def post(self, integration_name: str, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def put(self, integration_id: int, **kwargs): - with db.get_session() as session: - db_integration = session.query(IntegrationAdmin).where(IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - try: - settings = integration.settings_model.parse_obj(request.json) - except ValidationError as e: - return e.errors(), 400 + db_integration = IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + try: + settings = integration.settings_model.parse_obj(request.json) + except ValidationError as e: + return e.errors(), 400 - db_integration.settings = settings.dict() - db_integration.config = request.json.get('config') - db_integration.insert(session=session) + if request.json.get('is_default'): + db_integration.make_default() - if request.json.get('is_default'): - db_integration.make_default(session=session) - session.commit() - return IntegrationPD.from_orm(db_integration).dict(), 200 + db_integration.settings = settings.dict() + db_integration.config = request.json.get('config') + db_integration.insert() + return IntegrationPD.from_orm(db_integration).dict(), 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -236,15 +184,12 @@ def put(self, integration_id: int, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def patch(self, integration_id: int, **kwargs): - with db.get_session() as session: - db_integration: IntegrationAdmin = session.query(IntegrationAdmin).where( - IntegrationAdmin.id == integration_id).first() - integration = self.module.get_by_name(db_integration.name) - if not integration or not db_integration: - return {'error': 'integration not found'}, 404 - db_integration.make_default(session=session) - session.commit() - return {'msg': 'integration set as default'}, 200 + db_integration = IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).first() + integration = self.module.get_by_name(db_integration.name) + if not integration or not db_integration: + return {'error': 'integration not found'}, 404 + db_integration.make_default() + return {'msg': 'integration set as default'}, 200 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.delete"], @@ -254,28 +199,21 @@ def patch(self, integration_id: int, **kwargs): "developer": {"admin": False, "viewer": False, "editor": False}, }}) def delete(self, integration_id: int, **kwargs): - with db.get_session() as session: - del_id = session.query(IntegrationAdmin.id).where(IntegrationAdmin.id == integration_id).delete() - session.commit() - return del_id, 204 - + IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).delete() + IntegrationAdmin.commit() + return integration_id, 204 class API(api_tools.APIBase): url_params = [ '', '/', - '', '/', - '/', '//', - - '/', - '//', ] mode_handlers = { 'default': ProjectAPI, 'administration': AdminAPI, - } + } \ No newline at end of file diff --git a/models/integration.py b/models/integration.py index 86426bb..ae47fe5 100644 --- a/models/integration.py +++ b/models/integration.py @@ -7,6 +7,7 @@ from ..models.pd.integration import IntegrationBase + class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): __tablename__ = "integration" __table_args__ = ( @@ -29,43 +30,41 @@ class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL uid = Column(String(128), unique=True, nullable=False) - def make_default(self, session): - session.query(IntegrationAdmin).where( + def make_default(self): + IntegrationAdmin.query.filter( IntegrationAdmin.name == self.name, IntegrationAdmin.is_default == True, IntegrationAdmin.id != self.id ).update({IntegrationAdmin.is_default: False}) self.is_default = True - # session.add(self) - # session.commit() + super().insert() def set_task_id(self, task_id: str): - with db.get_session() as session: - session.query(IntegrationAdmin).where( - IntegrationAdmin.id == self.id - ).update({IntegrationAdmin.task_id: task_id}) - session.commit() + IntegrationAdmin.query.filter( + IntegrationAdmin.id == self.id + ).update({IntegrationAdmin.task_id: task_id}) + self.insert() - def insert(self, session): + def insert(self): if not self.uid: self.uid = str(uuid4()) - if not session.query(IntegrationAdmin).filter( - IntegrationAdmin.name == self.name, - IntegrationAdmin.is_default == True, - ).first(): + if not IntegrationAdmin.query.filter( + IntegrationAdmin.name == self.name, + IntegrationAdmin.is_default == True, + ).one_or_none(): self.is_default = True - session.add(self) - session.commit() - session.refresh(self) + super().insert() + self.process_secret_fields() + self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) + + def process_secret_fields(self): settings: dict = self.rpc.call.integrations_process_secrets( integration_data=IntegrationBase.from_orm(self).dict(), ) - session.query(IntegrationAdmin).where( + IntegrationAdmin.query.filter( IntegrationAdmin.id == self.id ).update({IntegrationAdmin.settings: settings}) - # session.commit() - self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) - + super().insert() class IntegrationProject(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): @@ -130,4 +129,4 @@ class IntegrationDefault(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin integration_id = Column(Integer, unique=False, nullable=False) project_id = Column(Integer, unique=False, nullable=True) is_default = Column(Boolean, default=False, nullable=False) - section = Column(String(64), unique=False, nullable=False) + section = Column(String(64), unique=False, nullable=False) \ No newline at end of file diff --git a/rpc/main.py b/rpc/main.py index cf95cb4..d1d9f2b 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -44,10 +44,6 @@ def get_by_name(self, integration_name: str) -> Optional[RegistrationForm]: def list_integrations(self) -> dict: return self.integrations - @rpc('list_integrations_by_section') - def list_integrations_by_section(self, section: str) -> list: - return [k for k, v in self.integrations.items() if v.section == section] - @rpc('get_project_integrations') def get_project_integrations(self, project_id: int, group_by_section: bool = True) -> dict: with db.with_project_schema_session(project_id) as tenant_session: @@ -136,52 +132,41 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ from administration mode :param integration_id: id of integration :return: integration ORM object or None - """ + """ if project_id is not None: with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationProject).filter( IntegrationProject.id == integration_id, - ).first() - with db.get_session() as session: - return session.query(IntegrationAdmin).where( - IntegrationAdmin.id == integration_id, - ).first() - + ).one_or_none() + return IntegrationAdmin.query.filter( + IntegrationAdmin.id == integration_id, + ).one_or_none() + @rpc('get_by_uid') - def get_by_uid( - self, integration_uid: str, - project_id: Optional[int] = None, - check_all_projects: bool = True - ) -> Optional[IntegrationProject]: + def get_by_uid(self, integration_uid: int, project_id: Optional[int] = None) -> Optional[IntegrationProject]: """ Get integration by unique id. You can specify current project_id but not necessary. :param integration_uid: uid of integration :param project_id: id of current project - :param check_all_projects: True - if we want to search in all projects :return: integration ORM object or None - """ - integration_uid = str(integration_uid) + """ if project_id is not None: - with db.get_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration + if integration := IntegrationAdmin.query.filter( + IntegrationAdmin.uid == integration_uid, + ).one_or_none(): + return integration + projects = self.context.rpc_manager.call.project_list() + for project in projects: + with db.with_project_schema_session(project['id']) as tenant_session: if integration := tenant_session.query(IntegrationProject).filter( IntegrationProject.uid == integration_uid, ).one_or_none(): - integration.project_id = project_id return integration - with db.get_session() as session: - if integration := session.query(IntegrationAdmin).where( - IntegrationAdmin.uid == integration_uid, - ).first(): - return integration - if check_all_projects: - projects = self.context.rpc_manager.call.project_list() - for project in projects: - with db.get_session(project['id']) as tenant_session: - if integration := tenant_session.query(IntegrationProject).where( - IntegrationProject.uid == integration_uid, - ).first(): - integration.project_id = project['id'] - return integration @web.rpc('security_test_create_integrations') @rpc_tools.wrap_exceptions(ValidationError) @@ -347,24 +332,23 @@ def get_cloud_integrations(self, project_id: int) -> list: return cloud_regions @rpc('get_administration_integrations') - def get_administration_integrations(self, group_by_section: bool = True) -> dict | List[IntegrationPD]: - with db.get_session() as session: - results = session.query(IntegrationAdmin).where( - IntegrationAdmin.name.in_(self.integrations.keys()) - ).group_by( - IntegrationAdmin.section, - IntegrationAdmin.id - ).order_by( - asc(IntegrationAdmin.section), - desc(IntegrationAdmin.is_default), - asc(IntegrationAdmin.name), - desc(IntegrationAdmin.id) - ).all() + def get_administration_integrations(self, group_by_section: bool = True) -> dict: + results = IntegrationAdmin.query.filter( + IntegrationAdmin.name.in_(self.integrations.keys()) + ).group_by( + IntegrationAdmin.section, + IntegrationAdmin.id + ).order_by( + asc(IntegrationAdmin.section), + desc(IntegrationAdmin.is_default), + asc(IntegrationAdmin.name), + desc(IntegrationAdmin.id) + ).all() - results = parse_obj_as(List[IntegrationPD], results) + results = parse_obj_as(List[IntegrationPD], results) - if not group_by_section: - return results + if not group_by_section: + return results def reducer(accum: dict, new_value: IntegrationPD) -> dict: accum[new_value.section.name].append(new_value) @@ -373,7 +357,7 @@ def reducer(accum: dict, new_value: IntegrationPD) -> dict: return reduce(reducer, results, defaultdict(list)) @rpc('get_administration_integrations_by_name') - def get_administration_integrations_by_name(self, integration_name: str, + def get_administration_integrations_by_name(self, integration_name: str, only_shared: bool = False ) -> List[IntegrationPD]: if integration_name not in self.integrations.keys(): @@ -417,12 +401,12 @@ def process_default_integrations(self, project_id, integrations): def _is_default(default_integrations, integration): for default_integration in default_integrations: - if (integration.project_id == default_integration.project_id and - integration.name == default_integration.name and + if (integration.project_id == default_integration.project_id and + integration.name == default_integration.name and integration.id == default_integration.integration_id ): return True - return False + return False for integration in integrations: integration.is_default = False @@ -481,26 +465,16 @@ def get_all_integrations_by_section(self, project_id: int, section_name: str) -> results_admin = self.get_administration_integrations_by_section(section_name, True) return self.process_default_integrations(project_id, results_project + results_admin) - @rpc('get_sorted_paginated_integrations_by_section') - def get_sorted_paginated_integrations_by_section(self, section_name: str, project_id: int, sort_order: str, - sort_by: str, offset: int, limit: int): - results_project = self.get_project_integrations_by_section(project_id, section_name) - results_admin = self.get_administration_integrations_by_section(section_name, True) - results = parse_obj_as(List[IntegrationPD], results_project + results_admin) - descending = sort_order.lower() == 'desc' - sorted_list = sorted(results, key=lambda x: getattr(x, sort_by), reverse=descending) - return sorted_list[offset:limit] - @rpc('update_attrs') - def update_attrs(self, - integration_id: int, - project_id: Optional[int], - update_dict: dict, + def update_attrs(self, + integration_id: int, + project_id: Optional[int], + update_dict: dict, return_result: bool = False ) -> Optional[dict]: update_dict.pop('id', None) if project_id: - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: log.info('update_attrs called %s', [integration_id, project_id, update_dict]) tenant_session.query(IntegrationProject).filter( IntegrationProject.id == integration_id @@ -518,7 +492,7 @@ def update_attrs(self, @rpc('make_default_integration') def make_default_integration(self, integration, project_id): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if default_integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration.name, IntegrationDefault.is_default == True, @@ -528,7 +502,7 @@ def make_default_integration(self, integration, project_id): tenant_session.commit() else: default_integration = IntegrationDefault(name=integration.name, - project_id=integration.project_id, + project_id=integration.project_id, integration_id = integration.id, is_default=True, section=integration.section @@ -538,7 +512,7 @@ def make_default_integration(self, integration, project_id): @rpc('delete_default_integration') def delete_default_integration(self, integration, project_id): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if default_integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration.name, IntegrationDefault.is_default == True, @@ -549,7 +523,7 @@ def delete_default_integration(self, integration, project_id): @rpc('get_defaults') def get_defaults(self, project_id, name=None): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if name: if integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == name, @@ -563,19 +537,19 @@ def get_defaults(self, project_id, name=None): def get_admin_defaults(self, name=None): if name: if integration := IntegrationAdmin.query.filter( - IntegrationAdmin.is_default == True, + IntegrationAdmin.is_default == True, IntegrationAdmin.name == name, ).one_or_none(): return IntegrationPD.from_orm(integration) else: results= IntegrationAdmin.query.filter( - IntegrationAdmin.is_default == True, + IntegrationAdmin.is_default == True, ).all() return parse_obj_as(List[IntegrationPD], results) @rpc('is_default') def is_default(self, project_id, integration_data): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration_data['name'], IntegrationDefault.is_default == True, @@ -596,13 +570,13 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): return _usecret_field(integration_db, project_id, is_local=True) elif integration_id: if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id, + IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True ).one_or_none(): return _usecret_field(integration_db, project_id, is_local=False) # in case if integration_id is not provided - try to find default integration: - else: + else: with db.with_project_schema_session(project_id) as tenant_session: default_integration = tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration_name @@ -630,12 +604,12 @@ def get_s3_admin_settings(self, integration_id=None): try: if integration_id: if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id, + IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, ).one_or_none(): return _usecret_field(integration_db, None, is_local=False) # in case if integration_id is not provided - try to find default integration: - else: + else: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.name == integration_name, IntegrationAdmin.is_default == True, @@ -660,4 +634,4 @@ def get_s3_admin_settings(self, integration_id=None): # section=integration_db.section # ) # tenant_session.add(default_integration) - # tenant_session.commit() + # tenant_session.commit() \ No newline at end of file