From ec0ef1b88f206f0cf754cd3c7b2fd91c36fa458b Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Tue, 7 Oct 2025 16:34:17 +0300 Subject: [PATCH 1/6] Generated and fixed up-to-date migrations --- .gitignore | 1 + ...1446_ab55e423b1e4_multicommit_fix_state.py | 364 ++++++++++++++++++ 2 files changed, 365 insertions(+) create mode 100644 migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py diff --git a/.gitignore b/.gitignore index 68bc17f9..f0f77ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -123,6 +123,7 @@ celerybeat.pid .env .venv env/ +venv venv/ ENV/ env.bak/ diff --git a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py new file mode 100644 index 00000000..1985b4e4 --- /dev/null +++ b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py @@ -0,0 +1,364 @@ +"""multicommit: fix state + +Revision ID: ab55e423b1e4 +Revises: 3ac342622c64 +Create Date: 2025-10-07 14:46:07.684999 + +""" + +import os + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'ab55e423b1e4' +down_revision = '3ac342622c64' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_weighted', + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Integer(), + existing_comment='Взвешенная оценка преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_kindness_weighted', + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Integer(), + existing_comment='Взвешенная доброта преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_clarity_weighted', + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Integer(), + existing_comment='Взвешенная понятность преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_freebie_weighted', + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Integer(), + existing_comment='Взвешенная халявность преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.drop_table_comment('dm_strike', existing_comment='\n Витрина страйки\n ', schema='DM_RENTAL') + op.drop_index(op.f('lecturer_ts_idx'), table_name='lecturer', schema='DWH_RATING') + + # МИГРАЦИЯ: Надо поменять на op.execute("... USING issue_id::Integer"); + op.alter_column( + 'git_hub', + 'issue_id', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + comment='Идентификатор issue', + existing_nullable=True, + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub', + 'user_id', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + comment='Идентификатор пользователя открывшего issue', + existing_nullable=True, + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub', + 'repository_id', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + comment='Идентификатор репозитория', + existing_nullable=True, + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub', + 'assignee_id', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + comment='Идентификатор назначенного исполнителем issue', + existing_nullable=True, + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub', + 'assignee_login', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + comment='Логин назначенного исполнителем issue', + existing_nullable=True, + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub', + 'organization_id', + existing_type=sa.VARCHAR(), + type_=sa.Integer(), + nullable=False, + comment='Идентификатор организации', + schema='ODS_SOCIAL', + postgresql_using='git_hub.issue_id::integer', + ) + op.alter_column( + 'git_hub_username', + 'username', + existing_type=sa.VARCHAR(), + comment='Имя пользователя GitHub', + existing_nullable=False, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'user_id', + existing_type=sa.INTEGER(), + comment='Идентификатор пользователя', + existing_nullable=False, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'source', + existing_type=sa.VARCHAR(), + comment='Источник данных', + existing_nullable=False, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'created', + existing_type=postgresql.TIMESTAMP(), + comment='Дата создания записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'modified', + existing_type=postgresql.TIMESTAMP(), + comment='Дата последнего изменения записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'is_deleted', + existing_type=sa.BOOLEAN(), + comment='Флаг удаления записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'union_member', + 'middle_name', + existing_type=sa.VARCHAR(), + comment='Отчество пользователя', + existing_nullable=True, + schema='STG_UNION_MEMBER', + ) + op.drop_column('union_member', 'card', schema='STG_UNION_MEMBER') + op.drop_table('test') + + +def downgrade(): + op.create_table( + 'test', + sa.Column('id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('val', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('test_pkey')), + ) + op.add_column( + 'union_member', sa.Column('card', sa.VARCHAR(), autoincrement=False, nullable=True), schema='STG_UNION_MEMBER' + ) + op.alter_column( + 'union_member', + 'middle_name', + existing_type=sa.VARCHAR(), + comment=None, + existing_comment='Отчество пользователя', + existing_nullable=True, + schema='STG_UNION_MEMBER', + ) + op.alter_column( + 'git_hub_username', + 'is_deleted', + existing_type=sa.BOOLEAN(), + comment=None, + existing_comment='Флаг удаления записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'modified', + existing_type=postgresql.TIMESTAMP(), + comment=None, + existing_comment='Дата последнего изменения записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'created', + existing_type=postgresql.TIMESTAMP(), + comment=None, + existing_comment='Дата создания записи', + existing_nullable=True, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'source', + existing_type=sa.VARCHAR(), + comment=None, + existing_comment='Источник данных', + existing_nullable=False, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'user_id', + existing_type=sa.INTEGER(), + comment=None, + existing_comment='Идентификатор пользователя', + existing_nullable=False, + schema='ODS_USERDATA', + ) + op.alter_column( + 'git_hub_username', + 'username', + existing_type=sa.VARCHAR(), + comment=None, + existing_comment='Имя пользователя GitHub', + existing_nullable=False, + schema='ODS_USERDATA', + ) + # При касте integer -> varchar, ручные изменения не нужны + op.alter_column( + 'git_hub', + 'organization_id', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + nullable=True, + comment=None, + existing_comment='Идентификатор организации', + schema='ODS_SOCIAL', + ) + op.alter_column( + 'git_hub', + 'assignee_login', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + comment=None, + existing_comment='Логин назначенного исполнителем issue', + existing_nullable=True, + schema='ODS_SOCIAL', + ) + op.alter_column( + 'git_hub', + 'assignee_id', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + comment=None, + existing_comment='Идентификатор назначенного исполнителем issue', + existing_nullable=True, + schema='ODS_SOCIAL', + ) + op.alter_column( + 'git_hub', + 'repository_id', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + comment=None, + existing_comment='Идентификатор репозитория', + existing_nullable=True, + schema='ODS_SOCIAL', + ) + op.alter_column( + 'git_hub', + 'user_id', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + comment=None, + existing_comment='Идентификатор пользователя открывшего issue', + existing_nullable=True, + schema='ODS_SOCIAL', + ) + op.alter_column( + 'git_hub', + 'issue_id', + existing_type=sa.Integer(), + type_=sa.VARCHAR(), + comment=None, + existing_comment='Идентификатор issue', + existing_nullable=True, + schema='ODS_SOCIAL', + ) + op.create_index( + op.f('lecturer_ts_idx'), 'lecturer', ['valid_from_dt', 'valid_to_dt'], unique=False, schema='DWH_RATING' + ) + op.create_table_comment('dm_strike', '\n Витрина страйки\n ', existing_comment=None, schema='DM_RENTAL') + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_freebie_weighted', + existing_type=sa.Integer(), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_comment='Взвешенная халявность преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_clarity_weighted', + existing_type=sa.Integer(), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_comment='Взвешенная понятность преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_kindness_weighted', + existing_type=sa.Integer(), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_comment='Взвешенная доброта преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) + op.alter_column( + 'dm_lecturer_comment_act', + 'mark_weighted', + existing_type=sa.Integer(), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_comment='Взвешенная оценка преподавателя', + existing_nullable=False, + existing_server_default=sa.text("'0'::double precision"), + schema='DM_RATING', + ) From f24e2454093240232da00a7716c8a5369b45b125 Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Thu, 9 Oct 2025 00:54:52 +0300 Subject: [PATCH 2/6] Implement 'sensitive' and 'encrypted' decorators --- profcomff_definitions/STG/userdata.py | 6 +- profcomff_definitions/base.py | 101 +++++++++++++++++++++----- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/profcomff_definitions/STG/userdata.py b/profcomff_definitions/STG/userdata.py index 60808710..cb7955aa 100644 --- a/profcomff_definitions/STG/userdata.py +++ b/profcomff_definitions/STG/userdata.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Mapped, mapped_column -from profcomff_definitions.base import Base, SensitiveBase +from profcomff_definitions.base import Base, sensitive, encrypted class Category(Base): @@ -50,11 +50,13 @@ class Info(Base): is_deleted: Mapped[bool | None] = mapped_column(comment="Флаг удаления записи") -class InfoKeys(SensitiveBase): +@sensitive +class InfoKeys(Base): id: Mapped[int] = mapped_column(primary_key=True, comment="Идентификатор ключа шифрования") key: Mapped[str] = mapped_column(comment="Симметричный ключ шифрования") +@encrypted("id", "InfoKeys") class EncryptedInfo(Base): id: Mapped[int] = mapped_column(primary_key=True, comment="Идентификатор зашифрованной записи") param_id: Mapped[int | None] = mapped_column(comment="Идентификатор параметра") diff --git a/profcomff_definitions/base.py b/profcomff_definitions/base.py index ad3ffd12..a49e03e6 100644 --- a/profcomff_definitions/base.py +++ b/profcomff_definitions/base.py @@ -3,7 +3,77 @@ from definitions.custom_scripts.schemas import add_table_schema_to_model from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import as_declarative +from sqlalchemy.orm import as_declarative, configure_mappers +from sqlalchemy import Table + + +def _recursive_merge(a: dict, b: dict) -> None: + """Helper function for recursive_merge""" + for key, val in b.items(): + if key in a and isinstance(a[key], dict) and isinstance(val, dict): + _recursive_merge(a[key], val) + continue + a[key] = val + + +def recursive_merge(a: dict, b: dict) -> dict: + """Returns a copy of 'a', recursively-updated with 'b' + (acts like recursive dict .update operator) + """ + ret = a.copy() + _recursive_merge(ret, b) + return ret + + +def sensitive(cls): + """Mark table as sensitive + + Sets 'info.sensitive' attribute to True. This attribute can be accessed in + definitions-lib in table comparator with + metadata_table_code.info.get("sensitive", False) + """ + new_table_args = recursive_merge(cls.__table_args__, {"info": {"sensitive": True}, "extend_existing": True}) + # Private function from sqlalchemy/sql/schema.py:Table + # Can be replaced with cls.__table__ = Table(...), but will be less optimal + cls.__table__._init_existing(**new_table_args) + return cls + + +def encrypted(id_column: str, key_table: str | None): + """Mark table as encrypted + + - id_column: encryption keys are going to be generated for all rows and + will be unique for each `id_column` value. Should be a name of primary + key in a table. + + - key_table (optional): table name to store keys in. If unspecified, + generated automatically (check generated migration scripts for the actual + name) + + Sets 'info.encrypted' attribute to True, and adds information about + encryption in 'info.encryption' attribute. + + Attributes can be accessesed in definitions-lib in table comparator with + metadata_table_code.info.get("attribute-name", default_value); + """ + + def encrypted_decorator(cls): + new_table_args = recursive_merge( + cls.__table_args__, + { + "info": { + # "sensitive": True, # Not needed since table is encrypted anyways + "encrypted": True, + "encryption": {"id": id_column, "key": key_table}, + }, + "extend_existing": True, + }, + ) + # See comment in function "sensitive" + cls.__table__._init_existing(**new_table_args) + return cls + + return encrypted_decorator @as_declarative() @@ -19,30 +89,23 @@ def __tablename__(cls) -> str: return re.sub(r"(? dict[str, Any]: schema = f'{cls.__module__.split(".")[-2].upper()}_{cls.__module__.split(".")[-1].upper()}' add_table_schema_to_model(schema, Base.metadata) - - return {'schema': schema, 'comment': cls.__doc__, 'info': {'sensitive': False}} + return_dict = { + 'schema': schema, + 'comment': cls.__doc__, + 'info': { + 'sensitive': getattr(cls, "__sensitive__", False), + 'encrypted': getattr(cls, "__encrypted__", False), + }, + } + recursive_merge(return_dict, getattr(cls, "__table_args_overwrite__", {})) + return return_dict def __repr__(self) -> str: attrs = [] for c in self.__table__.columns: attrs.append(f"{c.name}={getattr(self, c.name)}") return "{}({})".format(self.__class__.__name__, ", ".join(attrs)) - - -class SensitiveBase(Base): - """Base class for all sensitive entities""" - - @classmethod - @declared_attr - def __table_args__(cls) -> dict[str, Any]: - schema = f'{cls.__module__.split(".")[-2].upper()}_{cls.__module__.split(".")[-1].upper()}' - add_table_schema_to_model(schema, Base.metadata) - - return {'schema': schema, 'comment': cls.__doc__, 'info': {'sensitive': True}} - - __abstract__ = True - __mapper_args__ = {'concrete': True} From 7c9668d4644ea289e08afc51d840901d5a419936 Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Thu, 9 Oct 2025 00:59:41 +0300 Subject: [PATCH 3/6] cleanup --- ...1446_ab55e423b1e4_multicommit_fix_state.py | 1 + profcomff_definitions/STG/userdata.py | 2 +- profcomff_definitions/base.py | 22 +++++++++---------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py index 1985b4e4..fd58fddf 100644 --- a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py +++ b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py @@ -12,6 +12,7 @@ from alembic import op from sqlalchemy.dialects import postgresql + # revision identifiers, used by Alembic. revision = 'ab55e423b1e4' down_revision = '3ac342622c64' diff --git a/profcomff_definitions/STG/userdata.py b/profcomff_definitions/STG/userdata.py index cb7955aa..7ec20d44 100644 --- a/profcomff_definitions/STG/userdata.py +++ b/profcomff_definitions/STG/userdata.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Mapped, mapped_column -from profcomff_definitions.base import Base, sensitive, encrypted +from profcomff_definitions.base import Base, encrypted, sensitive class Category(Base): diff --git a/profcomff_definitions/base.py b/profcomff_definitions/base.py index a49e03e6..a5790933 100644 --- a/profcomff_definitions/base.py +++ b/profcomff_definitions/base.py @@ -3,8 +3,7 @@ from definitions.custom_scripts.schemas import add_table_schema_to_model from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import as_declarative, configure_mappers -from sqlalchemy import Table +from sqlalchemy.orm import as_declarative def _recursive_merge(a: dict, b: dict) -> None: @@ -32,10 +31,11 @@ def sensitive(cls): definitions-lib in table comparator with metadata_table_code.info.get("sensitive", False) """ - new_table_args = recursive_merge(cls.__table_args__, {"info": {"sensitive": True}, "extend_existing": True}) + cls.__table_args__ = recursive_merge(cls.__table_args__, {"info": {"sensitive": True}, "extend_existing": True}) # Private function from sqlalchemy/sql/schema.py:Table # Can be replaced with cls.__table__ = Table(...), but will be less optimal - cls.__table__._init_existing(**new_table_args) + cls.__table__._init_existing(**cls.__table_args__) + return cls @@ -58,7 +58,7 @@ def encrypted(id_column: str, key_table: str | None): """ def encrypted_decorator(cls): - new_table_args = recursive_merge( + cls.__table_args__ = recursive_merge( cls.__table_args__, { "info": { @@ -70,7 +70,7 @@ def encrypted_decorator(cls): }, ) # See comment in function "sensitive" - cls.__table__._init_existing(**new_table_args) + cls.__table__._init_existing(**cls.__table_args__) return cls return encrypted_decorator @@ -89,20 +89,18 @@ def __tablename__(cls) -> str: return re.sub(r"(? dict[str, Any]: schema = f'{cls.__module__.split(".")[-2].upper()}_{cls.__module__.split(".")[-1].upper()}' add_table_schema_to_model(schema, Base.metadata) - return_dict = { + return { 'schema': schema, 'comment': cls.__doc__, 'info': { - 'sensitive': getattr(cls, "__sensitive__", False), - 'encrypted': getattr(cls, "__encrypted__", False), + 'sensitive': False, + 'encrypted': False, }, } - recursive_merge(return_dict, getattr(cls, "__table_args_overwrite__", {})) - return return_dict def __repr__(self) -> str: attrs = [] From 376fe83b790ecc3ed289d78aa693e0b788c8cc6e Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Mon, 3 Nov 2025 23:48:11 +0300 Subject: [PATCH 4/6] cleanup base, removed unused table, fix migration --- ...1446_ab55e423b1e4_multicommit_fix_state.py | 8 ---- profcomff_definitions/STG/userdata.py | 15 ++------ profcomff_definitions/base.py | 38 ++++++++++++------- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py index fd58fddf..0646fd68 100644 --- a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py +++ b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py @@ -182,16 +182,8 @@ def upgrade(): schema='STG_UNION_MEMBER', ) op.drop_column('union_member', 'card', schema='STG_UNION_MEMBER') - op.drop_table('test') - def downgrade(): - op.create_table( - 'test', - sa.Column('id', sa.INTEGER(), autoincrement=False, nullable=False), - sa.Column('val', sa.INTEGER(), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('test_pkey')), - ) op.add_column( 'union_member', sa.Column('card', sa.VARCHAR(), autoincrement=False, nullable=True), schema='STG_UNION_MEMBER' ) diff --git a/profcomff_definitions/STG/userdata.py b/profcomff_definitions/STG/userdata.py index 7ec20d44..55b9cae6 100644 --- a/profcomff_definitions/STG/userdata.py +++ b/profcomff_definitions/STG/userdata.py @@ -39,6 +39,8 @@ class Source(Base): is_deleted: Mapped[bool | None] = mapped_column(comment="Флаг удаления записи") +# uncomment when corresponding functionality is implemented in definitions-lib +# @encrypted(columns=['value']) class Info(Base): id: Mapped[int] = mapped_column(primary_key=True, comment="Идентификатор информационной записи") param_id: Mapped[int | None] = mapped_column(comment="Идентификатор параметра") @@ -54,15 +56,4 @@ class Info(Base): class InfoKeys(Base): id: Mapped[int] = mapped_column(primary_key=True, comment="Идентификатор ключа шифрования") key: Mapped[str] = mapped_column(comment="Симметричный ключ шифрования") - - -@encrypted("id", "InfoKeys") -class EncryptedInfo(Base): - id: Mapped[int] = mapped_column(primary_key=True, comment="Идентификатор зашифрованной записи") - param_id: Mapped[int | None] = mapped_column(comment="Идентификатор параметра") - source_id: Mapped[int | None] = mapped_column(comment="Идентификатор источника данных") - owner_id: Mapped[int | None] = mapped_column(comment="Идентификатор владельца данных") - value: Mapped[bytes | None] = mapped_column(comment="Зашифрованное значение параметра") - create_ts: Mapped[datetime | None] = mapped_column(comment="Время создания записи") - modify_ts: Mapped[datetime | None] = mapped_column(comment="Время последнего изменения записи") - is_deleted: Mapped[bool | None] = mapped_column(comment="Флаг удаления записи") + created: Mapped[datetime] = mapped_column(comment="Дата создания/обновления ключа") diff --git a/profcomff_definitions/base.py b/profcomff_definitions/base.py index a5790933..e3b60476 100644 --- a/profcomff_definitions/base.py +++ b/profcomff_definitions/base.py @@ -39,33 +39,44 @@ def sensitive(cls): return cls -def encrypted(id_column: str, key_table: str | None): +def encrypted(id_column: str = "id", **kwargs): """Mark table as encrypted - - id_column: encryption keys are going to be generated for all rows and - will be unique for each `id_column` value. Should be a name of primary - key in a table. + Arguments: + - id_column (optional, default: "id"): encryption keys are going to be + generated for all rows and will be unique for each `id_column` value. + Should be a name of primary key in a table. - - key_table (optional): table name to store keys in. If unspecified, - generated automatically (check generated migration scripts for the actual - name) + Keyword arguments: + - key_table (default: _ekeys): table name to store keys in. If + unspecified, generated automatically (check generated migration scripts + for the actual name) + - columns (default: all columns except id_column): array of column names to + be encrypted + + Action: Sets 'info.encrypted' attribute to True, and adds information about encryption in 'info.encryption' attribute. + Example: + # key_table defaults to "user_info_ekeys" + @encrypted('user_id', columns=['email', 'name']) + class UserInfo(Base): + ... + Attributes can be accessesed in definitions-lib in table comparator with - metadata_table_code.info.get("attribute-name", default_value); + metadata_table_.info.get("attribute-name", default_value); """ def encrypted_decorator(cls): + _key_table = kwargs.get('key_table', cls.__tablename__ + "_ekeys") + _columns = kwargs.get('columns', [i.name for i in cls.__table__.columns if i.name != id_column]) + cls.__table_args__ = recursive_merge( cls.__table_args__, { - "info": { - # "sensitive": True, # Not needed since table is encrypted anyways - "encrypted": True, - "encryption": {"id": id_column, "key": key_table}, - }, + "info": {"encrypted": True, "encryption": {"id": id_column, "keys": _key_table, "columns": _columns}}, "extend_existing": True, }, ) @@ -92,6 +103,7 @@ def __tablename__(cls) -> str: @declared_attr def __table_args__(cls) -> dict[str, Any]: schema = f'{cls.__module__.split(".")[-2].upper()}_{cls.__module__.split(".")[-1].upper()}' + # this line is very important for definitions-lib! Without it, the library wouldn't know which schemas are present in the database add_table_schema_to_model(schema, Base.metadata) return { 'schema': schema, From 89191728a6d58e93f001b8c093fa73e99f7f5cc4 Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Sat, 15 Nov 2025 00:02:09 +0300 Subject: [PATCH 5/6] Formatting fixes --- .../20251007_1446_ab55e423b1e4_multicommit_fix_state.py | 1 + profcomff_definitions/DM/rental.py | 2 +- profcomff_definitions/DWH/rental.py | 2 +- profcomff_definitions/ODS/userdata.py | 2 +- profcomff_definitions/STG/userdata.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py index 0646fd68..0efd8215 100644 --- a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py +++ b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py @@ -183,6 +183,7 @@ def upgrade(): ) op.drop_column('union_member', 'card', schema='STG_UNION_MEMBER') + def downgrade(): op.add_column( 'union_member', sa.Column('card', sa.VARCHAR(), autoincrement=False, nullable=True), schema='STG_UNION_MEMBER' diff --git a/profcomff_definitions/DM/rental.py b/profcomff_definitions/DM/rental.py index a4cbd7b9..cd7f8df7 100644 --- a/profcomff_definitions/DM/rental.py +++ b/profcomff_definitions/DM/rental.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timedelta +from datetime import datetime from uuid import UUID from sqlalchemy.orm import Mapped, mapped_column diff --git a/profcomff_definitions/DWH/rental.py b/profcomff_definitions/DWH/rental.py index 3c401c79..b675f0aa 100644 --- a/profcomff_definitions/DWH/rental.py +++ b/profcomff_definitions/DWH/rental.py @@ -1,7 +1,7 @@ from datetime import date, datetime from uuid import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column from profcomff_definitions.base import Base diff --git a/profcomff_definitions/ODS/userdata.py b/profcomff_definitions/ODS/userdata.py index b1391aad..9cb82a79 100644 --- a/profcomff_definitions/ODS/userdata.py +++ b/profcomff_definitions/ODS/userdata.py @@ -1,4 +1,4 @@ -from datetime import date, datetime +from datetime import datetime from sqlalchemy.orm import Mapped, mapped_column diff --git a/profcomff_definitions/STG/userdata.py b/profcomff_definitions/STG/userdata.py index 55b9cae6..c2cb4b57 100644 --- a/profcomff_definitions/STG/userdata.py +++ b/profcomff_definitions/STG/userdata.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Mapped, mapped_column -from profcomff_definitions.base import Base, encrypted, sensitive +from profcomff_definitions.base import Base, sensitive class Category(Base): From e2bd6e99b637bac77b578202f6769e15fe3d7f09 Mon Sep 17 00:00:00 2001 From: justanothercatgirl Date: Sat, 15 Nov 2025 00:20:37 +0300 Subject: [PATCH 6/6] Apply definition-lib functionality --- ...250315_1949_5edf7206774e_fix_redirector.py | 2 - ...1c_added_column_to_fetch_with_back_data.py | 2 - ...0250415_2142_5c9e53fd0e21_rental_errors.py | 2 - ...50415_2154_40d72f2dd950_fix_viribus_ods.py | 2 - .../20250416_0827_d43d62fb3748_fix_chat.py | 1 - .../versions/20250417_1932_b9c1ac6c4360_.py | 6 - ...34_018159bb0cfc_add_col_in_dm_marketing.py | 2 - .../versions/20250421_1620_32883c034be8_.py | 6 - ...dwh_pipelines_177_added_keys_table_and_.py | 2 - ...a18_added_dwh_user_info_info_encrypted_.py | 2 - .../20250429_2314_83a86d9e5c9b_rental_fix.py | 3 - ...1311_1d108dc04935_drop_table_dwh_rental.py | 2 - .../20250522_1336_3c2c26d09969_ods_gitstat.py | 2 - ...524_2352_31293ac08820_rental_struct_fix.py | 2 - ...259a3b8c31d_ods_social_id_data_type_fix.py | 2 - ...8_1802_d1a22199f353_dm_rental_fix_table.py | 2 - ...50705_1543_80e114fe1399_bot_actions_fix.py | 2 - ...50804_1833_5345485db41c_student_id_info.py | 2 - ...20250807_1713_dd8e7cd6c56e_ods_userdata.py | 1 - ...50817_0032_059a2a1571f1_rating_refactor.py | 2 - ..._2a352723139d_comments_stg_union_member.py | 3 - ...c633eace8bc_add_new_tables_ods_userdata.py | 2 - ...24_1a383420f47f_ods_userdata_card_table.py | 3 - ..._dm_rental_refactor_and_strike_datamart.py | 2 - ...1446_ab55e423b1e4_multicommit_fix_state.py | 2 - ...d7c58_apply_new_definitions_lib_changes.py | 436 ++++++++++++++++++ 26 files changed, 436 insertions(+), 59 deletions(-) create mode 100644 migrations/versions/20251115_0012_af84d0ad7c58_apply_new_definitions_lib_changes.py diff --git a/migrations/versions/20250315_1949_5edf7206774e_fix_redirector.py b/migrations/versions/20250315_1949_5edf7206774e_fix_redirector.py index b24c0c35..1ebc5189 100644 --- a/migrations/versions/20250315_1949_5edf7206774e_fix_redirector.py +++ b/migrations/versions/20250315_1949_5edf7206774e_fix_redirector.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250412_1504_066fd36bb01c_added_column_to_fetch_with_back_data.py b/migrations/versions/20250412_1504_066fd36bb01c_added_column_to_fetch_with_back_data.py index 045fec5d..098798fb 100644 --- a/migrations/versions/20250412_1504_066fd36bb01c_added_column_to_fetch_with_back_data.py +++ b/migrations/versions/20250412_1504_066fd36bb01c_added_column_to_fetch_with_back_data.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250415_2142_5c9e53fd0e21_rental_errors.py b/migrations/versions/20250415_2142_5c9e53fd0e21_rental_errors.py index 6b56cc84..a7ede5c9 100644 --- a/migrations/versions/20250415_2142_5c9e53fd0e21_rental_errors.py +++ b/migrations/versions/20250415_2142_5c9e53fd0e21_rental_errors.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250415_2154_40d72f2dd950_fix_viribus_ods.py b/migrations/versions/20250415_2154_40d72f2dd950_fix_viribus_ods.py index a68fbcec..561e8fd3 100644 --- a/migrations/versions/20250415_2154_40d72f2dd950_fix_viribus_ods.py +++ b/migrations/versions/20250415_2154_40d72f2dd950_fix_viribus_ods.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20250416_0827_d43d62fb3748_fix_chat.py b/migrations/versions/20250416_0827_d43d62fb3748_fix_chat.py index 4921505a..a07e9531 100644 --- a/migrations/versions/20250416_0827_d43d62fb3748_fix_chat.py +++ b/migrations/versions/20250416_0827_d43d62fb3748_fix_chat.py @@ -8,7 +8,6 @@ import os -import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250417_1932_b9c1ac6c4360_.py b/migrations/versions/20250417_1932_b9c1ac6c4360_.py index 92950e53..2823953b 100644 --- a/migrations/versions/20250417_1932_b9c1ac6c4360_.py +++ b/migrations/versions/20250417_1932_b9c1ac6c4360_.py @@ -6,12 +6,6 @@ """ -import os - -import sqlalchemy as sa -from alembic import op - - # revision identifiers, used by Alembic. revision = 'b9c1ac6c4360' down_revision = ('5c9e53fd0e21', '40d72f2dd950') diff --git a/migrations/versions/20250417_1934_018159bb0cfc_add_col_in_dm_marketing.py b/migrations/versions/20250417_1934_018159bb0cfc_add_col_in_dm_marketing.py index 9a36876e..b544e7d7 100644 --- a/migrations/versions/20250417_1934_018159bb0cfc_add_col_in_dm_marketing.py +++ b/migrations/versions/20250417_1934_018159bb0cfc_add_col_in_dm_marketing.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250421_1620_32883c034be8_.py b/migrations/versions/20250421_1620_32883c034be8_.py index 8fdec4bd..b8efc7bd 100644 --- a/migrations/versions/20250421_1620_32883c034be8_.py +++ b/migrations/versions/20250421_1620_32883c034be8_.py @@ -6,12 +6,6 @@ """ -import os - -import sqlalchemy as sa -from alembic import op - - # revision identifiers, used by Alembic. revision = '32883c034be8' down_revision = ('d43d62fb3748', '018159bb0cfc') diff --git a/migrations/versions/20250424_2354_7baa38ad06f1_dwh_pipelines_177_added_keys_table_and_.py b/migrations/versions/20250424_2354_7baa38ad06f1_dwh_pipelines_177_added_keys_table_and_.py index 0110b8bd..b551fef6 100644 --- a/migrations/versions/20250424_2354_7baa38ad06f1_dwh_pipelines_177_added_keys_table_and_.py +++ b/migrations/versions/20250424_2354_7baa38ad06f1_dwh_pipelines_177_added_keys_table_and_.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250426_1710_00dc9eb31a18_added_dwh_user_info_info_encrypted_.py b/migrations/versions/20250426_1710_00dc9eb31a18_added_dwh_user_info_info_encrypted_.py index 526eb2c7..768bf5a2 100644 --- a/migrations/versions/20250426_1710_00dc9eb31a18_added_dwh_user_info_info_encrypted_.py +++ b/migrations/versions/20250426_1710_00dc9eb31a18_added_dwh_user_info_info_encrypted_.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250429_2314_83a86d9e5c9b_rental_fix.py b/migrations/versions/20250429_2314_83a86d9e5c9b_rental_fix.py index ffe0c5d9..6f286cf4 100644 --- a/migrations/versions/20250429_2314_83a86d9e5c9b_rental_fix.py +++ b/migrations/versions/20250429_2314_83a86d9e5c9b_rental_fix.py @@ -6,11 +6,8 @@ """ -import os - import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/migrations/versions/20250509_1311_1d108dc04935_drop_table_dwh_rental.py b/migrations/versions/20250509_1311_1d108dc04935_drop_table_dwh_rental.py index 18422577..5f142612 100644 --- a/migrations/versions/20250509_1311_1d108dc04935_drop_table_dwh_rental.py +++ b/migrations/versions/20250509_1311_1d108dc04935_drop_table_dwh_rental.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20250522_1336_3c2c26d09969_ods_gitstat.py b/migrations/versions/20250522_1336_3c2c26d09969_ods_gitstat.py index 9ffe634a..c0ac1939 100644 --- a/migrations/versions/20250522_1336_3c2c26d09969_ods_gitstat.py +++ b/migrations/versions/20250522_1336_3c2c26d09969_ods_gitstat.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250524_2352_31293ac08820_rental_struct_fix.py b/migrations/versions/20250524_2352_31293ac08820_rental_struct_fix.py index 70e54209..69b64ce9 100644 --- a/migrations/versions/20250524_2352_31293ac08820_rental_struct_fix.py +++ b/migrations/versions/20250524_2352_31293ac08820_rental_struct_fix.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20250528_1331_b259a3b8c31d_ods_social_id_data_type_fix.py b/migrations/versions/20250528_1331_b259a3b8c31d_ods_social_id_data_type_fix.py index e1a68f11..6ca2b5d1 100644 --- a/migrations/versions/20250528_1331_b259a3b8c31d_ods_social_id_data_type_fix.py +++ b/migrations/versions/20250528_1331_b259a3b8c31d_ods_social_id_data_type_fix.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250528_1802_d1a22199f353_dm_rental_fix_table.py b/migrations/versions/20250528_1802_d1a22199f353_dm_rental_fix_table.py index d672c5cb..d4e33369 100644 --- a/migrations/versions/20250528_1802_d1a22199f353_dm_rental_fix_table.py +++ b/migrations/versions/20250528_1802_d1a22199f353_dm_rental_fix_table.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250705_1543_80e114fe1399_bot_actions_fix.py b/migrations/versions/20250705_1543_80e114fe1399_bot_actions_fix.py index 126e1f23..2c4c2be5 100644 --- a/migrations/versions/20250705_1543_80e114fe1399_bot_actions_fix.py +++ b/migrations/versions/20250705_1543_80e114fe1399_bot_actions_fix.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250804_1833_5345485db41c_student_id_info.py b/migrations/versions/20250804_1833_5345485db41c_student_id_info.py index 94ffea50..b0dc6327 100644 --- a/migrations/versions/20250804_1833_5345485db41c_student_id_info.py +++ b/migrations/versions/20250804_1833_5345485db41c_student_id_info.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/20250807_1713_dd8e7cd6c56e_ods_userdata.py b/migrations/versions/20250807_1713_dd8e7cd6c56e_ods_userdata.py index 1ae00237..d540d1e9 100644 --- a/migrations/versions/20250807_1713_dd8e7cd6c56e_ods_userdata.py +++ b/migrations/versions/20250807_1713_dd8e7cd6c56e_ods_userdata.py @@ -10,7 +10,6 @@ import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql revision = 'dd8e7cd6c56e' diff --git a/migrations/versions/20250817_0032_059a2a1571f1_rating_refactor.py b/migrations/versions/20250817_0032_059a2a1571f1_rating_refactor.py index 5a367d12..ab942e67 100644 --- a/migrations/versions/20250817_0032_059a2a1571f1_rating_refactor.py +++ b/migrations/versions/20250817_0032_059a2a1571f1_rating_refactor.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20250909_1508_2a352723139d_comments_stg_union_member.py b/migrations/versions/20250909_1508_2a352723139d_comments_stg_union_member.py index 44ceac36..65c0a50f 100644 --- a/migrations/versions/20250909_1508_2a352723139d_comments_stg_union_member.py +++ b/migrations/versions/20250909_1508_2a352723139d_comments_stg_union_member.py @@ -6,11 +6,8 @@ """ -import os - import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql revision = '2a352723139d' diff --git a/migrations/versions/20250909_1648_6c633eace8bc_add_new_tables_ods_userdata.py b/migrations/versions/20250909_1648_6c633eace8bc_add_new_tables_ods_userdata.py index a96640a2..a6e45027 100644 --- a/migrations/versions/20250909_1648_6c633eace8bc_add_new_tables_ods_userdata.py +++ b/migrations/versions/20250909_1648_6c633eace8bc_add_new_tables_ods_userdata.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20250918_2124_1a383420f47f_ods_userdata_card_table.py b/migrations/versions/20250918_2124_1a383420f47f_ods_userdata_card_table.py index 2eeaae39..7e32c01d 100644 --- a/migrations/versions/20250918_2124_1a383420f47f_ods_userdata_card_table.py +++ b/migrations/versions/20250918_2124_1a383420f47f_ods_userdata_card_table.py @@ -6,11 +6,8 @@ """ -import os - import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql revision = '1a383420f47f' diff --git a/migrations/versions/20250919_2126_3ac342622c64_dm_rental_refactor_and_strike_datamart.py b/migrations/versions/20250919_2126_3ac342622c64_dm_rental_refactor_and_strike_datamart.py index 068d6b93..74430f26 100644 --- a/migrations/versions/20250919_2126_3ac342622c64_dm_rental_refactor_and_strike_datamart.py +++ b/migrations/versions/20250919_2126_3ac342622c64_dm_rental_refactor_and_strike_datamart.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py index 0efd8215..364936c7 100644 --- a/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py +++ b/migrations/versions/20251007_1446_ab55e423b1e4_multicommit_fix_state.py @@ -6,8 +6,6 @@ """ -import os - import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql diff --git a/migrations/versions/20251115_0012_af84d0ad7c58_apply_new_definitions_lib_changes.py b/migrations/versions/20251115_0012_af84d0ad7c58_apply_new_definitions_lib_changes.py new file mode 100644 index 00000000..e001f5fb --- /dev/null +++ b/migrations/versions/20251115_0012_af84d0ad7c58_apply_new_definitions_lib_changes.py @@ -0,0 +1,436 @@ +"""Apply new definitions-lib changes + +Revision ID: af84d0ad7c58 +Revises: ab55e423b1e4 +Create Date: 2025-11-15 00:12:36.907971 + +""" + +import os + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision = 'af84d0ad7c58' +down_revision = 'ab55e423b1e4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_group( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ) + op.create_group( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ) + op.create_group( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ) + op.grant_on_schema( + ( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ), + "STG_USERDATA", + ) + op.grant_on_schema( + ( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ), + "STG_USERDATA", + ) + op.grant_on_schema( + ( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ), + "STG_USERDATA", + ) + op.grant_on_table( + "test_dwh_dm_rental_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_read", + ['SELECT'], + '"DM_RENTAL".dm_strike', + ) + op.grant_on_table( + "test_dwh_dm_rental_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_all", + ['ALL'], + '"DM_RENTAL".dm_strike', + ) + op.grant_on_table( + "test_dwh_dm_rental_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_RENTAL".dm_strike', + ) + op.grant_on_table( + "test_dwh_dm_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.grant_on_table( + "test_dwh_dm_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_read", + ['SELECT'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.grant_on_table( + "test_dwh_dm_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_all", + ['ALL'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.grant_on_table( + "test_dwh_dm_user_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_USER".union_member_card', + ) + op.grant_on_table( + "test_dwh_dm_user_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_all", + ['ALL'], + '"DM_USER".union_member_card', + ) + op.grant_on_table( + "test_dwh_dm_user_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_read", + ['SELECT'], + '"DM_USER".union_member_card', + ) + op.grant_on_table( + "test_dwh_dwh_user_info_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_all", + ['ALL'], + '"DWH_USER_INFO".encrypted_info', + ) + op.grant_on_table( + "test_dwh_dwh_user_info_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DWH_USER_INFO".encrypted_info', + ) + op.grant_on_table( + "test_dwh_dwh_user_info_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_read", + ['SELECT'], + '"DWH_USER_INFO".encrypted_info', + ) + op.grant_on_table( + "test_dwh_ods_social_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_read", + ['SELECT'], + '"ODS_SOCIAL".git_hub', + ) + op.grant_on_table( + "test_dwh_ods_social_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_SOCIAL".git_hub', + ) + op.grant_on_table( + "test_dwh_ods_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_all", + ['ALL'], + '"ODS_SOCIAL".git_hub', + ) + op.grant_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".card', + ) + op.grant_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".card', + ) + op.grant_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".card', + ) + op.grant_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".rzd', + ) + op.grant_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".rzd', + ) + op.grant_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".rzd', + ) + op.grant_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".status', + ) + op.grant_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".status', + ) + op.grant_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".status', + ) + op.add_column( + 'info_keys', + sa.Column('created', sa.DateTime(), nullable=False, comment='Дата создания/обновления ключа'), + schema='STG_USERDATA', + ) + op.grant_on_table( + ( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ), + ['ALL'], + '"STG_USERDATA".info_keys', + ) + op.grant_on_table( + ( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ), + ['SELECT'], + '"STG_USERDATA".info_keys', + ) + op.grant_on_table( + ( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ), + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"STG_USERDATA".info_keys', + ) + op.drop_table('encrypted_info', schema='STG_USERDATA') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'encrypted_info', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False, comment='Идентификатор зашифрованной записи'), + sa.Column('param_id', sa.INTEGER(), autoincrement=False, nullable=True, comment='Идентификатор параметра'), + sa.Column( + 'source_id', sa.INTEGER(), autoincrement=False, nullable=True, comment='Идентификатор источника данных' + ), + sa.Column( + 'owner_id', sa.INTEGER(), autoincrement=False, nullable=True, comment='Идентификатор владельца данных' + ), + sa.Column( + 'value', postgresql.BYTEA(), autoincrement=False, nullable=True, comment='Зашифрованное значение параметра' + ), + sa.Column( + 'create_ts', postgresql.TIMESTAMP(), autoincrement=False, nullable=True, comment='Время создания записи' + ), + sa.Column( + 'modify_ts', + postgresql.TIMESTAMP(), + autoincrement=False, + nullable=True, + comment='Время последнего изменения записи', + ), + sa.Column('is_deleted', sa.BOOLEAN(), autoincrement=False, nullable=True, comment='Флаг удаления записи'), + sa.PrimaryKeyConstraint('id', name=op.f('encrypted_info_pkey')), + schema='STG_USERDATA', + ) + op.revoke_on_table( + ( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ), + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"STG_USERDATA".info_keys', + ) + op.revoke_on_table( + ( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ), + ['SELECT'], + '"STG_USERDATA".info_keys', + ) + op.revoke_on_table( + ( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ), + ['ALL'], + '"STG_USERDATA".info_keys', + ) + op.drop_column('info_keys', 'created', schema='STG_USERDATA') + op.revoke_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".status', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".status', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".status', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".rzd', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".rzd', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".rzd', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_USERDATA".card', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_all", + ['ALL'], + '"ODS_USERDATA".card', + ) + op.revoke_on_table( + "test_dwh_ods_userdata_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_userdata_read", + ['SELECT'], + '"ODS_USERDATA".card', + ) + op.revoke_on_table( + "test_dwh_ods_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_all", + ['ALL'], + '"ODS_SOCIAL".git_hub', + ) + op.revoke_on_table( + "test_dwh_ods_social_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"ODS_SOCIAL".git_hub', + ) + op.revoke_on_table( + "test_dwh_ods_social_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_social_read", + ['SELECT'], + '"ODS_SOCIAL".git_hub', + ) + op.revoke_on_table( + "test_dwh_dwh_user_info_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_read", + ['SELECT'], + '"DWH_USER_INFO".encrypted_info', + ) + op.revoke_on_table( + "test_dwh_dwh_user_info_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DWH_USER_INFO".encrypted_info', + ) + op.revoke_on_table( + "test_dwh_dwh_user_info_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dwh_user_info_all", + ['ALL'], + '"DWH_USER_INFO".encrypted_info', + ) + op.revoke_on_table( + "test_dwh_dm_user_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_read", + ['SELECT'], + '"DM_USER".union_member_card', + ) + op.revoke_on_table( + "test_dwh_dm_user_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_all", + ['ALL'], + '"DM_USER".union_member_card', + ) + op.revoke_on_table( + "test_dwh_dm_user_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_user_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_USER".union_member_card', + ) + op.revoke_on_table( + "test_dwh_dm_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_all", + ['ALL'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.revoke_on_table( + "test_dwh_dm_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_read", + ['SELECT'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.revoke_on_table( + "test_dwh_dm_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_timetable_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_TIMETABLE".dm_timetable_act', + ) + op.revoke_on_table( + "test_dwh_dm_rental_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_write", + ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], + '"DM_RENTAL".dm_strike', + ) + op.revoke_on_table( + "test_dwh_dm_rental_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_all", + ['ALL'], + '"DM_RENTAL".dm_strike', + ) + op.revoke_on_table( + "test_dwh_dm_rental_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_rental_read", + ['SELECT'], + '"DM_RENTAL".dm_strike', + ) + op.revoke_on_schema( + ( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ), + "STG_USERDATA", + ) + op.revoke_on_schema( + ( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ), + "STG_USERDATA", + ) + op.revoke_on_schema( + ( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ), + "STG_USERDATA", + ) + op.delete_group( + "test_sensitive_dwh_stg_userdata_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_read" + ) + op.delete_group( + "test_sensitive_dwh_stg_userdata_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_write" + ) + op.delete_group( + "test_sensitive_dwh_stg_userdata_all" + if os.getenv("ENVIRONMENT") != "production" + else "prod_sensitive_dwh_stg_userdata_all" + ) + # ### end Alembic commands ###