diff --git a/api/v1/available.py b/api/v1/available.py new file mode 100644 index 0000000..29d1d61 --- /dev/null +++ b/api/v1/available.py @@ -0,0 +1,31 @@ +from flask import request + +from tools import auth, api_tools + + +class ProjectAPI(api_tools.APIModeHandler): + ... + + +class AdminAPI(api_tools.APIModeHandler): + ... + + +class API(api_tools.APIBase): + url_params = [ + '', + '/', + '', + '/' + ] + + mode_handlers = { + 'default': ProjectAPI, + 'administration': AdminAPI, + } + + def get(self, **kwargs): + section = request.args.get('section') + if section: + return self.module.list_integrations_by_section(section), 200 + return list(self.module.list_integrations()), 200 diff --git a/api/v1/check_settings.py b/api/v1/check_settings.py index ee8892b..389f5a1 100644 --- a/api/v1/check_settings.py +++ b/api/v1/check_settings.py @@ -23,20 +23,27 @@ class API(api_tools.APIBase): 'administration': AdminAPI, } - @auth.decorators.check_api(["configuration.integrations.integrations.create", - "configuration.integrations.integrations.edit" - ]) + @auth.decorators.check_api( + [ + "configuration.integrations.integrations.create", + "configuration.integrations.integrations.edit" + ], + project_id_in_request_json=True + ) def post(self, integration_name: str, **kwargs): integration = self.module.get_by_name(integration_name) + payload = request.json if not integration: return {'error': 'integration not found'}, 404 try: - settings = integration.settings_model.parse_obj(request.json) + settings = integration.settings_model.parse_obj(payload) except ValidationError as e: # return e.json(), 400 return e.errors(), 400 - check_connection_response = settings.check_connection() + project_id = payload.get('project_id') + project_id = int(project_id) if project_id else project_id + check_connection_response = settings.check_connection(project_id) if not request.json.get('save_action'): if check_connection_response is True: return 'OK', 200 diff --git a/api/v1/integration.py b/api/v1/integration.py index db39dae..8da2ea2 100644 --- a/api/v1/integration.py +++ b/api/v1/integration.py @@ -43,7 +43,7 @@ def post(self, integration_name: str): try: return IntegrationPD.from_orm(db_integration).dict(), 200 except ValidationError as e: - return e.errors(), 400 + return e.errors(), 400 @auth.decorators.check_api({ "permissions": ["configuration.integrations.integrations.edit"], @@ -108,7 +108,7 @@ def patch(self, project_id: int, integration_id: int): "administration": {"admin": True, "viewer": False, "editor": False}, "default": {"admin": True, "viewer": False, "editor": False}, "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + }}) def delete(self, project_id: int, integration_id: int): with db.with_project_schema_session(project_id) as tenant_session: db_integration = tenant_session.query(IntegrationProject).filter( @@ -197,7 +197,7 @@ def patch(self, integration_id: int, **kwargs): "administration": {"admin": True, "viewer": False, "editor": False}, "default": {"admin": True, "viewer": False, "editor": False}, "developer": {"admin": False, "viewer": False, "editor": False}, - }}) + }}) def delete(self, integration_id: int, **kwargs): IntegrationAdmin.query.filter(IntegrationAdmin.id == integration_id).delete() IntegrationAdmin.commit() @@ -216,4 +216,4 @@ class API(api_tools.APIBase): mode_handlers = { 'default': ProjectAPI, 'administration': AdminAPI, - } + } \ No newline at end of file diff --git a/api/v1/integrations.py b/api/v1/integrations.py index 6d43d36..481a55d 100644 --- a/api/v1/integrations.py +++ b/api/v1/integrations.py @@ -48,6 +48,28 @@ def get(self, **kwargs): ], 200 +class PromptLibAPI(api_tools.APIModeHandler): + AI_SECTION: str = 'ai' + + @auth.decorators.check_api({ + "permissions": ["configuration.integrations.integrations.view"], + "recommended_roles": { + "administration": {"admin": True, "viewer": True, "editor": True}, + "default": {"admin": True, "viewer": True, "editor": True}, + "developer": {"admin": True, "viewer": False, "editor": False}, + }}) + def get(self, project_id: int): + sort_order = request.args.get('sort_order', 'asc') + sort_by = request.args.get('sort_by', 'name') + offset = int(request.args.get('offset', 0)) + limit = int(request.args.get('limit', 10_000)) + return [ + i.dict() for i in self.module.get_sorted_paginated_integrations_by_section( + self.AI_SECTION, project_id, sort_order, sort_by, offset, limit + ) + ], 200 + + class API(api_tools.APIBase): url_params = [ '', @@ -59,4 +81,5 @@ class API(api_tools.APIBase): mode_handlers = { 'default': ProjectAPI, 'administration': AdminAPI, + 'prompt_lib': PromptLibAPI, } diff --git a/events/__init__.py b/events/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/events/main.py b/events/main.py new file mode 100644 index 0000000..ed5ae15 --- /dev/null +++ b/events/main.py @@ -0,0 +1,37 @@ +from sqlalchemy import Boolean + +from ..models.integration import IntegrationAdmin, IntegrationDefault +from ..models.pd.integration import SecretField + +from tools import rpc_tools, VaultClient, db + +from pylon.core.tools import web, log + + +def _usecret_field(integration_db, project_id): + settings = integration_db.settings + secret_access_key = SecretField.parse_obj(settings['secret_access_key']) + settings['secret_access_key'] = secret_access_key.unsecret(project_id=project_id) + return settings + + +class Event: + @web.event('project_created') + def create_default_s3_for_new_project(self, context, event, project: dict, **kwargs) -> None: + log.info('Creating default integration for project %s', project) + project_id = project['id'] + if integration_db := IntegrationAdmin.query.filter( + IntegrationAdmin.name == 's3_integration', + IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, + IntegrationAdmin.is_default == True, + ).one_or_none(): + with db.with_project_schema_session(project_id) as tenant_session: + default_integration = IntegrationDefault( + name=integration_db.name, + project_id=None, + integration_id=integration_db.id, + is_default=True, + section=integration_db.section + ) + tenant_session.add(default_integration) + tenant_session.commit() diff --git a/models/integration.py b/models/integration.py index d8410e1..ae47fe5 100644 --- a/models/integration.py +++ b/models/integration.py @@ -1,6 +1,7 @@ from pylon.core.tools import log from sqlalchemy import Integer, Column, String, Boolean, UniqueConstraint, Index from sqlalchemy.dialects.postgresql import JSON +from uuid import uuid4 from tools import db_tools, db, rpc_tools from ..models.pd.integration import IntegrationBase @@ -19,15 +20,15 @@ class IntegrationAdmin(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, ) id = Column(Integer, primary_key=True) name = Column(String(64), unique=False) - # project_id = Column(Integer, unique=False, nullable=True) - # mode = Column(String(64), unique=False, default='default') settings = Column(JSON, unique=False, default={}) is_default = Column(Boolean, default=False, nullable=False) section = Column(String(64), unique=False, nullable=False) - # description = Column(String(256), unique=False, nullable=True, default='Default integration') config = Column(JSON, unique=False, default={}) task_id = Column(String(256), unique=False, nullable=True) status = Column(String(256), unique=False, nullable=False, default='success') + # ALTER TABLE "Project-1"."integration" ADD COLUMN uid VARCHAR(128) + # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL + uid = Column(String(128), unique=True, nullable=False) def make_default(self): IntegrationAdmin.query.filter( @@ -45,6 +46,8 @@ def set_task_id(self, task_id: str): self.insert() def insert(self): + if not self.uid: + self.uid = str(uuid4()) if not IntegrationAdmin.query.filter( IntegrationAdmin.name == self.name, IntegrationAdmin.is_default == True, @@ -66,50 +69,24 @@ def process_secret_fields(self): class IntegrationProject(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin, rpc_tools.EventManagerMixin): __tablename__ = "integration" - # __table_args__ = ( - # Index( - # 'ix_project_default_uc', # Index name - # 'project_id', 'name', # Columns which are part of the index - # unique=True, - # postgresql_where=Column('is_default') # The condition - # ) - # ) __table_args__ = {'schema': 'tenant'} id = Column(Integer, primary_key=True) name = Column(String(64), unique=False) project_id = Column(Integer, unique=False, nullable=True) - # mode = Column(String(64), unique=False, default='default') settings = Column(JSON, unique=False, default={}) is_default = Column(Boolean, default=False, nullable=False) section = Column(String(64), unique=False, nullable=False) - # description = Column(String(256), unique=False, nullable=True, default='Default integration') config = Column(JSON, unique=False, default={}) task_id = Column(String(256), unique=False, nullable=True) status = Column(String(256), unique=False, nullable=False, default='success') - - - # def make_default(self, session): - # default_integration = session.query(IntegrationProject).filter( - # IntegrationProject.project_id == self.project_id, - # IntegrationProject.name == self.name, - # IntegrationProject.is_default == True, - # IntegrationProject.id != self.id - # ).one_or_none() - # if default_integration: - # default_integration.is_default = False - # self.is_default = True - # # super().insert() - # session.commit() - - # def set_task_id(self, session, task_id: str): - # session.query(IntegrationProject).filter( - # IntegrationProject.id == self.id - # ).update({IntegrationProject.task_id: task_id}) - # # self.insert() - # session.commit() + # ALTER TABLE "Project-1"."integration" ADD COLUMN uid VARCHAR(128) + # ALTER TABLE "Project-1"."integration" ALTER COLUMN uid NOT NULL + uid = Column(String(128), unique=True, nullable=False) def insert(self, session): + if not self.uid: + self.uid = str(uuid4()) session.add(self) session.commit() inherited_integration = IntegrationAdmin.query.filter( @@ -122,7 +99,6 @@ def insert(self, session): ).one_or_none() if not inherited_integration and not default_integration: self.rpc.call.integrations_make_default_integration(self, self.project_id) - # super().insert(session) self.process_secret_fields(session) self.event_manager.fire_event(f'{self.name}_created_or_updated', self.to_json()) @@ -133,7 +109,6 @@ def process_secret_fields(self, session): session.query(IntegrationProject).filter( IntegrationProject.id == self.id ).update({IntegrationProject.settings: settings}) - # super().insert() session.commit() @@ -154,4 +129,4 @@ class IntegrationDefault(db_tools.AbstractBaseMixin, db.Base, rpc_tools.RpcMixin integration_id = Column(Integer, unique=False, nullable=False) project_id = Column(Integer, unique=False, nullable=True) is_default = Column(Boolean, default=False, nullable=False) - section = Column(String(64), unique=False, nullable=False) + section = Column(String(64), unique=False, nullable=False) \ No newline at end of file diff --git a/models/pd/integration.py b/models/pd/integration.py index f0c3175..8c4f95c 100644 --- a/models/pd/integration.py +++ b/models/pd/integration.py @@ -1,4 +1,5 @@ from typing import Optional, Union +from uuid import uuid4 from pydantic import BaseModel, validator, constr from pylon.core.tools import log @@ -18,13 +19,19 @@ class IntegrationBase(BaseModel): config: dict task_id: Optional[str] status: Optional[str] = 'success' - # mode: str + uid: str class Config: orm_mode = True class IntegrationPD(IntegrationBase): + @validator('uid', pre=True, always=True) + def set_uid(cls, value: Optional[str]): + if not value: + return str(uuid4()) + return value + @validator("settings") def validate_settings(cls, value, values): integration = rpc_tools.RpcMixin().rpc.call.integrations_get_by_name( @@ -44,11 +51,6 @@ def validate_section(cls, value, values): return rpc_tools.RpcMixin().rpc.call.integrations_register_section(name=value) return section - # @validator("config") - # def validate_config(cls, value, values): - # assert value.get('name'), 'ensure this value has at least 1 characters' - # return value - @validator("config") def validate_description(cls, value, values): if not value.get('name'): @@ -57,15 +59,6 @@ def validate_description(cls, value, values): return value -# class IntegrationProjectPD(IntegrationPD): -# pass - # @validator("is_default") - # def validate_is_default(cls, value, values): - # if rpc_tools.RpcMixin().rpc.call.integrations_is_default(values['project_id'], values): - # return True - # return False - - class IntegrationDefaultPD(BaseModel): id: int name: str diff --git a/module.py b/module.py index 02004f8..65417c3 100644 --- a/module.py +++ b/module.py @@ -19,8 +19,8 @@ from pylon.core.tools import log # pylint: disable=E0611,E0401 from pylon.core.tools import module -from .models.integration import IntegrationAdmin # pylint: disable=E0611,E0401 -from .models.pd.integration import IntegrationBase +# from .models.integration import IntegrationAdmin # pylint: disable=E0611,E0401 +# from .models.pd.integration import IntegrationBase from .init_db import init_db @@ -46,6 +46,7 @@ def init(self): self.descriptor.init_blueprint() self.descriptor.init_api() self.descriptor.init_slots() + self.descriptor.init_events() theme.register_subsection( 'configuration', 'integrations', diff --git a/rpc/main.py b/rpc/main.py index 3728fd2..d1d9f2b 100644 --- a/rpc/main.py +++ b/rpc/main.py @@ -17,10 +17,12 @@ from pylon.core.tools import web -def _usecret_field(integration_db, project_id): +def _usecret_field(integration_db, project_id, is_local): settings = integration_db.settings secret_access_key = SecretField.parse_obj(settings['secret_access_key']) settings['secret_access_key'] = secret_access_key.unsecret(project_id=project_id) + settings['integration_id'] = integration_db.id + settings['is_local'] = is_local return settings @@ -124,6 +126,13 @@ def section_list(self) -> list: @rpc('get_by_id') def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[IntegrationProject]: + """ + Get integration by id. Works properly if you know: inherited this integration or not. + :param project_id: id of project, where integration was created. If None - integration + from administration mode + :param integration_id: id of integration + :return: integration ORM object or None + """ if project_id is not None: with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationProject).filter( @@ -132,7 +141,32 @@ def get_by_id(self, project_id: Optional[int], integration_id: int) -> Optional[ return IntegrationAdmin.query.filter( IntegrationAdmin.id == integration_id, ).one_or_none() - + + @rpc('get_by_uid') + def get_by_uid(self, integration_uid: int, project_id: Optional[int] = None) -> Optional[IntegrationProject]: + """ + Get integration by unique id. You can specify current project_id but not necessary. + :param integration_uid: uid of integration + :param project_id: id of current project + :return: integration ORM object or None + """ + if project_id is not None: + with db.with_project_schema_session(project_id) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration + if integration := IntegrationAdmin.query.filter( + IntegrationAdmin.uid == integration_uid, + ).one_or_none(): + return integration + projects = self.context.rpc_manager.call.project_list() + for project in projects: + with db.with_project_schema_session(project['id']) as tenant_session: + if integration := tenant_session.query(IntegrationProject).filter( + IntegrationProject.uid == integration_uid, + ).one_or_none(): + return integration @web.rpc('security_test_create_integrations') @rpc_tools.wrap_exceptions(ValidationError) @@ -323,7 +357,7 @@ def reducer(accum: dict, new_value: IntegrationPD) -> dict: return reduce(reducer, results, defaultdict(list)) @rpc('get_administration_integrations_by_name') - def get_administration_integrations_by_name(self, integration_name: str, + def get_administration_integrations_by_name(self, integration_name: str, only_shared: bool = False ) -> List[IntegrationPD]: if integration_name not in self.integrations.keys(): @@ -367,12 +401,12 @@ def process_default_integrations(self, project_id, integrations): def _is_default(default_integrations, integration): for default_integration in default_integrations: - if (integration.project_id == default_integration.project_id and - integration.name == default_integration.name and + if (integration.project_id == default_integration.project_id and + integration.name == default_integration.name and integration.id == default_integration.integration_id ): return True - return False + return False for integration in integrations: integration.is_default = False @@ -432,15 +466,15 @@ def get_all_integrations_by_section(self, project_id: int, section_name: str) -> return self.process_default_integrations(project_id, results_project + results_admin) @rpc('update_attrs') - def update_attrs(self, - integration_id: int, - project_id: Optional[int], - update_dict: dict, + def update_attrs(self, + integration_id: int, + project_id: Optional[int], + update_dict: dict, return_result: bool = False ) -> Optional[dict]: update_dict.pop('id', None) if project_id: - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: log.info('update_attrs called %s', [integration_id, project_id, update_dict]) tenant_session.query(IntegrationProject).filter( IntegrationProject.id == integration_id @@ -458,7 +492,7 @@ def update_attrs(self, @rpc('make_default_integration') def make_default_integration(self, integration, project_id): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if default_integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration.name, IntegrationDefault.is_default == True, @@ -468,7 +502,7 @@ def make_default_integration(self, integration, project_id): tenant_session.commit() else: default_integration = IntegrationDefault(name=integration.name, - project_id=integration.project_id, + project_id=integration.project_id, integration_id = integration.id, is_default=True, section=integration.section @@ -478,7 +512,7 @@ def make_default_integration(self, integration, project_id): @rpc('delete_default_integration') def delete_default_integration(self, integration, project_id): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if default_integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration.name, IntegrationDefault.is_default == True, @@ -489,7 +523,7 @@ def delete_default_integration(self, integration, project_id): @rpc('get_defaults') def get_defaults(self, project_id, name=None): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: if name: if integration := tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == name, @@ -503,19 +537,19 @@ def get_defaults(self, project_id, name=None): def get_admin_defaults(self, name=None): if name: if integration := IntegrationAdmin.query.filter( - IntegrationAdmin.is_default == True, + IntegrationAdmin.is_default == True, IntegrationAdmin.name == name, ).one_or_none(): return IntegrationPD.from_orm(integration) else: results= IntegrationAdmin.query.filter( - IntegrationAdmin.is_default == True, + IntegrationAdmin.is_default == True, ).all() return parse_obj_as(List[IntegrationPD], results) @rpc('is_default') def is_default(self, project_id, integration_data): - with db.with_project_schema_session(project_id) as tenant_session: + with db.with_project_schema_session(project_id) as tenant_session: return tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration_data['name'], IntegrationDefault.is_default == True, @@ -533,16 +567,16 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): IntegrationProject.id == integration_id, IntegrationProject.name == integration_name ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=True) elif integration_id: if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id, + IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=False) # in case if integration_id is not provided - try to find default integration: - else: + else: with db.with_project_schema_session(project_id) as tenant_session: default_integration = tenant_session.query(IntegrationDefault).filter( IntegrationDefault.name == integration_name @@ -552,17 +586,17 @@ def get_s3_settings(self, project_id, integration_id=None, is_local=True): IntegrationProject.id == default_integration.integration_id, IntegrationProject.name == integration_name ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=True) elif default_integration: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.id == default_integration.integration_id, IntegrationAdmin.name == integration_name, IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True ).one_or_none(): - return _usecret_field(integration_db, project_id) + return _usecret_field(integration_db, project_id, is_local=False) except Exception as e: log.warning(f'Cannot receive S3 settings for project {project_id}') - log.warning(e) + log.debug(e) @rpc('get_s3_admin_settings') def get_s3_admin_settings(self, integration_id=None): @@ -570,34 +604,34 @@ def get_s3_admin_settings(self, integration_id=None): try: if integration_id: if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.id == integration_id, + IntegrationAdmin.id == integration_id, IntegrationAdmin.name == integration_name, ).one_or_none(): - return _usecret_field(integration_db, None) + return _usecret_field(integration_db, None, is_local=False) # in case if integration_id is not provided - try to find default integration: - else: + else: if integration_db := IntegrationAdmin.query.filter( IntegrationAdmin.name == integration_name, IntegrationAdmin.is_default == True, ).one_or_none(): - return _usecret_field(integration_db, None) + return _usecret_field(integration_db, None, is_local=False) except Exception as e: log.warning(f'Cannot receive S3 settings in administration mode') - log.warning(e) - - @rpc('create_default_s3_for_new_project') - def create_default_s3_for_new_project(self, project_id): - if integration_db := IntegrationAdmin.query.filter( - IntegrationAdmin.name == 's3_integration', - IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, - IntegrationAdmin.is_default == True, - ).one_or_none(): - with db.with_project_schema_session(project_id) as tenant_session: - default_integration = IntegrationDefault(name=integration_db.name, - project_id=None, - integration_id = integration_db.id, - is_default=True, - section=integration_db.section - ) - tenant_session.add(default_integration) - tenant_session.commit() + log.debug(e) + + # @rpc('create_default_s3_for_new_project') + # def create_default_s3_for_new_project(self, project_id): + # if integration_db := IntegrationAdmin.query.filter( + # IntegrationAdmin.name == 's3_integration', + # IntegrationAdmin.config['is_shared'].astext.cast(Boolean) == True, + # IntegrationAdmin.is_default == True, + # ).one_or_none(): + # with db.with_project_schema_session(project_id) as tenant_session: + # default_integration = IntegrationDefault(name=integration_db.name, + # project_id=None, + # integration_id = integration_db.id, + # is_default=True, + # section=integration_db.section + # ) + # tenant_session.add(default_integration) + # tenant_session.commit() \ No newline at end of file diff --git a/static/js/security_app_create.js b/static/js/security_app_create.js index 6795f8f..03f2c60 100644 --- a/static/js/security_app_create.js +++ b/static/js/security_app_create.js @@ -161,7 +161,8 @@ const TestIntegrationItem = { }, methods: { getIntegrationTitle(integration) { - return integration.is_default ? `${integration.description} - default` : integration.description + integrationName = integration.hasOwnProperty('config')? integration.config.name : integration.description + return integration.is_default ? `${integrationName} - default` : integrationName }, clear_data() { this.is_selected = false diff --git a/static/js/security_code_create.js b/static/js/security_code_create.js index bbb96e3..109fa19 100644 --- a/static/js/security_code_create.js +++ b/static/js/security_code_create.js @@ -153,7 +153,8 @@ const TestIntegrationItem = { }, methods: { getIntegrationTitle(integration) { - return integration.is_default ? `${integration.description} - default` : integration.description + integrationName = integration.hasOwnProperty('config')? integration.config.name : integration.description + return integration.is_default ? `${integrationName} - default` : integrationName }, clear_data() { this.is_selected = false