Sync 2024-11-29

auth/auth_server/alembic/versions/76fd6db54f65_user_verified_column.py

@@ -0,0 +1,50 @@
+# pylint: disable=C0103
+"""user verified column
+
+Revision ID: 76fd6db54f65
+Revises: 245d4295bbd6
+Create Date: 2024-11-18 09:57:42.103381
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import update, Integer, and_, Boolean
+from sqlalchemy.orm import Session
+from sqlalchemy.sql import table, column
+
+# revision identifiers, used by Alembic.
+revision = '76fd6db54f65'
+down_revision = '245d4295bbd6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('user', sa.Column('verified', Boolean(), nullable=False))
+    user_table = table(
+        'user',
+        column('deleted_at', Integer()),
+        column('verified', Boolean())
+    )
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    try:
+        upd_stmt = update(user_table).values(verified=True).where(and_(
+            user_table.c.verified.is_(False),
+            user_table.c.deleted_at == 0
+        ))
+        session.execute(upd_stmt)
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('user', 'verified')
+    # ### end Alembic commands ###

auth/auth_server/controllers/signin.py

@@ -258,7 +258,7 @@ def signin(self, **kwargs):
                 password=self._gen_password(),
                 self_registration=True, token='',
                 is_password_autogenerated=True)
-
+        user.verified = True
         token_dict = self.token_ctl.create_token_by_user_id(
             user_id=user.id, ip=ip, provider=provider,
             register=register)

auth/auth_server/controllers/token.py

@@ -57,10 +57,14 @@ def _check_user(self, email, password=None, verification_code=None):
         if password:
             if user.password != hash_password(password, user.salt):
                 raise ForbiddenException(Err.OA0037, [])
+            if not user.verified:
+                raise ForbiddenException(Err.OA0073, [])
         else:
             vc_used = self.use_verification_code(email, verification_code)
             if not vc_used:
                 raise ForbiddenException(Err.OA0071, [])
+            elif not user.verified:
+                user.verified = True
         if not user.is_active:
             raise ForbiddenException(Err.OA0038, [])
         return user
@@ -94,6 +98,8 @@ def create_token_by_user_id(self, **kwargs):
             raise NotFoundException(Err.OA0043, [user_id])
         if not user.is_active:
             raise ForbiddenException(Err.OA0038, [])
+        if not user.verified:
+            raise ForbiddenException(Err.OA0073, [])
         return self.create_user_token(user, **kwargs)
 
     def create_user_token(self, user, **kwargs):

auth/auth_server/controllers/user.py

@@ -37,6 +37,7 @@ def _get_input(self, **input_):
         is_password_autogenerated = input_.get(
             'is_password_autogenerated', False)
         self_registration = input_.get('self_registration', False)
+        verified = input_.get('verified', False)
         if self_registration:
             root_type = self.session.query(Type).filter(
                 and_(
@@ -48,7 +49,7 @@ def _get_input(self, **input_):
                 raise NotFoundException(Err.OA0064, [])
             type_id = root_type.id
         return (email, display_name, is_active, password, type_id, scope_id,
-                is_password_autogenerated)
+                is_password_autogenerated, verified)
 
     @staticmethod
     def _is_self_edit(user, user_to_edit_id):
@@ -139,7 +140,7 @@ def create(self, **kwargs):
         self_registration = kwargs.pop('self_registration', False)
         self.check_create_restrictions(**kwargs)
         (email, display_name, is_active, password, type_id,
-         scope_id, is_password_autogenerated) = self._get_input(
+         scope_id, is_password_autogenerated, verified) = self._get_input(
             self_registration=self_registration, **kwargs)
         self._check_input(email, display_name, is_active, password, type_id,
                           scope_id)
@@ -163,7 +164,8 @@ def create(self, **kwargs):
             type_id=type_id,
             display_name=display_name,
             is_active=is_active,
-            is_password_autogenerated=is_password_autogenerated
+            is_password_autogenerated=is_password_autogenerated,
+            verified=verified
         )
         self.session.add(user)
         try:

auth/auth_server/exceptions.py

@@ -202,3 +202,6 @@ class Err(enum.Enum):
     OA0072 = [
         "The verification code can be generated once in a minute"
     ]
+    OA0073 = [
+        "Email not verified"
+    ]

auth/auth_server/handlers/v2/users.py

@@ -238,6 +238,8 @@ async def post(self, **url_params):
                     scope_id: {type: string,
                         description: User scope id (None scope means root)}
                     token: {type: string, description: Token}
+                    verified: {type: boolean,
+                        description: "Is email verified?"}
         responses:
             201: {description: Success (returns created user data)}
             400:
@@ -288,6 +290,9 @@ async def post(self, **url_params):
             unexpected_string = ', '.join(duplicates)
             raise OptHTTPError(400, Err.OA0022, [unexpected_string])
         body.update(url_params)
+        if not self.check_cluster_secret(raises=False):
+            # verified users can be created only by secret
+            body.pop('verified', None)
         self._validate_params(**body)
         res = await run_task(
             self.controller.create, **body, **self.token,

auth/auth_server/models/models.py

@@ -176,6 +176,8 @@ class User(Base, BaseMixin):
                      info=ColumnPermissions.create_only)
     email = Column(String(256), nullable=False, index=True,
                    info=ColumnPermissions.create_only)
+    verified = Column(Boolean, nullable=False, default=False,
+                      info=ColumnPermissions.create_only)
     password = Column(String(64), nullable=False,
                       info=ColumnPermissions.full)
     salt = Column(String(20), nullable=False)
@@ -204,7 +206,8 @@ def to_json(self):
     def __init__(self, email=None, type_=None, password=None,
                  salt=None, scope_id=None, type_id=None, display_name=None,
                  is_active=True, slack_connected=False,
-                 is_password_autogenerated=False, jira_connected=False):
+                 is_password_autogenerated=False, jira_connected=False,
+                 verified=False):
         if type_:
             self.type = type_
         if type_id is not None:
@@ -219,6 +222,7 @@ def __init__(self, email=None, type_=None, password=None,
         self.slack_connected = slack_connected
         self.is_password_autogenerated = is_password_autogenerated
         self.jira_connected = jira_connected
+        self.verified = verified
 
     def __repr__(self):
         return f'<User {self.email}>'

auth/auth_server/tests/unittests/test_api_base.py

@@ -58,8 +58,13 @@ def setUp(self, version='v1'):
         self.client = TestAuthBase.get_auth_client(version).Client(
             http_provider=http_provider)
         self.client.secret = secret
+        self.user_verified_mock = patch(
+            'auth.auth_server.models.models.User.verified',
+            new_callable=PropertyMock, return_value=True)
+        self.user_verified_mock.start()
 
     def tearDown(self):
+        self.user_verified_mock.stop()
         DBFactory.clean_type(DBType.TEST)
         super().tearDown()
 

auth/auth_server/tests/unittests/test_api_signin.py

@@ -241,3 +241,21 @@
                                             token='token')
         self.assertEqual(code, 201)
         self.assertEqual(resp['user_email'], self.user_customer_email)
+
+    @patch.dict(os.environ, {'MICROSOFT_OAUTH_CLIENT_ID': '123'}, clear=True)
+    def test_signin_user_verified(self):
+        self.user_verified_mock.stop()
+        patch('auth.auth_server.controllers.user.UserController.'
+              'domain_blacklist').start()
+        token_info = (self.user_customer_email, self.user_customer_name)
+        with patch('auth.auth_server.controllers.signin.'
+                   'MicrosoftOauth2Provider.verify',
+                   return_value=token_info):
+            code, resp = self.client.signin(provider='microsoft',
+                                            token='token')
+        self.assertEqual(code, 201)
+
+        code, resp = self.client.user_get(resp['user_id'])
+        self.assertEqual(code, 200)
+        self.assertEqual(resp['email'], self.user_customer_email)
+        self.assertTrue(resp['verified'])

auth/auth_server/tests/unittests/test_api_token.py

@@ -115,6 +115,7 @@ def test_get_token_cluster_secret(self):
         extract_caveats(token_info['token'])
 
     def test_token_by_verification_code(self):
+        self.user_verified_mock.stop()
         session = self.db_session
         partner_salt = gen_salt()
         partner_password = 'pass1234'
@@ -152,7 +153,9 @@ def test_token_by_verification_code(self):
             code, resp = self.client.post('tokens', body)
             self.assertEqual(code, 403)
             self.assertEqual(resp['error']['error_code'], 'OA0071')
-
+        verified = session.query(User.verified).filter(
+            User.id == partner_user.id).scalar()
+        self.assertFalse(verified)
         body = {
             'verification_code': code_4,
             'email': partner_user.email,
@@ -161,3 +164,6 @@ def test_token_by_verification_code(self):
         self.assertEqual(code, 201)
         self.assertEqual(resp['user_email'], partner_user.email)
         self.assertTrue(vc_4.deleted_at != 0)
+        verified = session.query(User.verified).filter(
+            User.id == partner_user.id).scalar()
+        self.assertTrue(verified)

auth/auth_server/tests/unittests/test_api_user.py

@@ -857,3 +857,18 @@ def test_create_email_domain_whitelist(self):
         wl_email = 'user@example.com'
         code, response = self._create_user(email=wl_email)
         self.assertEqual(code, 400)
+
+    def test_create_verified_user(self):
+        secret = self.client.secret
+        self.user_verified_mock.stop()
+        self.client.secret = None
+        code, user = self.client.user_create('test@email.com', 'pass1',
+                                             display_name='test',
+                                             verified=True)
+        self.assertFalse(user['verified'])
+
+        self.client.secret = secret
+        code, user = self.client.user_create('test2@email.com', 'pass1',
+                                             display_name='test',
+                                             verified=True)
+        self.assertTrue(user['verified'])

bumiworker/bumiworker/modules/archive/obsolete_snapshots.py

@@ -25,10 +25,8 @@ def supported_cloud_types(self):
 
     def get_used_resources(self, now, cloud_account_id, cloud_config,
                            obsolete_threshold):
-        snapshots_used_by_images = {}
-        if cloud_config.get('type') == AWS_CLOUD:
-            snapshots_used_by_images = self.get_snapshots_used_by_images(
-                now, cloud_config)
+        snapshots_used_by_images = self.get_snapshots_used_by_images(
+            now, cloud_config)
         snapshots_using_volumes = self._get_snapshots_using_volumes(
             now, cloud_account_id)
         snapshots_used_by_volumes = self.get_snapshots_used_by_volumes(

bumiworker/bumiworker/modules/obsolete_snapshots_base.py

@@ -35,6 +35,8 @@ def get_obsolete_resources(self, now, cloud_account_id, config,
 
     def merge_last_used(self, collection, to_merge):
         for resource_id, last_used in to_merge.items():
+            if resource_id is None:
+                continue
             current_last_used = collection.get(resource_id)
             if current_last_used is None or current_last_used < last_used:
                 collection[resource_id] = last_used
@@ -74,7 +76,8 @@ def get_resources_pipeline(self, resource_type, excluded_ids,
             }
         ]
 
-    def get_snapshots_used_by_images(self, now, cloud_config):
+    @staticmethod
+    def _get_snapshots_used_by_images_from_cloud(now, cloud_config):
         adapter = CloudAdapter.get_adapter(cloud_config)
         images = []
         snapshot_ids = set()
@@ -95,6 +98,25 @@ def get_snapshots_used_by_images(self, now, cloud_config):
                     snapshot_ids.add(snapshot_id)
         return {s: now for s in snapshot_ids}
 
+    def _get_snapshots_used_by_images_from_db(self, now, cloud_config):
+        snapshot_ids = self.mongo_client.restapi.resources.find({
+            'cloud_account_id': cloud_config['id'],
+            'resource_type': 'Image',
+            'active': True}, {'meta.snapshot_id': 1})
+        return {s['meta.snapshot_id']: now for s in snapshot_ids
+                if 'meta.snapshot_id' in s}
+
+    def get_snapshots_used_by_images(self, now, cloud_config):
+        cloud_funcs = {
+            'alibaba_cnr': self._get_snapshots_used_by_images_from_cloud,
+            'aws_cnr': self._get_snapshots_used_by_images_from_cloud,
+            'gcp_cnr': self._get_snapshots_used_by_images_from_db,
+        }
+        func = cloud_funcs.get(cloud_config.get('type'))
+        if not func:
+            return {}
+        return func(now, cloud_config)
+
     def get_snapshots_used_by_volumes(self, now, cloud_account_id,
                                       obsolete_threshold):
         last_seen = int((now - obsolete_threshold).timestamp())

bumiworker/bumiworker/modules/recommendations/obsolete_ips.py

@@ -7,6 +7,7 @@
     'aws_cnr',
     'azure_cnr',
     'alibaba_cnr',
+    'gcp_cnr',
     'nebius'
 ]
 

bumiworker/bumiworker/modules/recommendations/obsolete_snapshots.py

@@ -7,6 +7,7 @@
 AWS_CLOUD = 'aws_cnr'
 SUPPORTED_CLOUDS = [
     'aws_cnr',
+    'gcp_cnr',
     'nebius',
 ]
 
@@ -32,10 +33,8 @@ def get_supported_clouds(self):
 
     def get_obsolete_resources(self, now, cloud_account_id, config,
                                obsolete_threshold):
-        snapshots_used_by_images = {}
-        if config.get('type') == AWS_CLOUD:
-            snapshots_used_by_images = self.get_snapshots_used_by_images(
-                now, config)
+        snapshots_used_by_images = self.get_snapshots_used_by_images(
+            now, config)
 
         snapshots_used_by_volumes = self.get_snapshots_used_by_volumes(
             now, cloud_account_id, obsolete_threshold)

diworker/diworker/importers/gcp.py

@@ -165,6 +165,8 @@ def _get_resource_type_and_name(expense):
         if cost_type == 'regular':
             if 'Snapshot' in sku:
                 r_type = 'Snapshot'
+            elif 'Image' in sku:
+                r_type = 'Image'
             elif 'PD Capacity' in sku:
                 r_type = 'Volume'
             elif 'Storage' in sku:

diworker/diworker/migrations/2024111309000000_gcp_image_resource_type.py

@@ -0,0 +1,49 @@
+import logging
+from pymongo import UpdateOne
+from optscale_client.rest_api_client.client_v2 import Client as RestClient
+from diworker.diworker.migrations.base import BaseMigration
+"""
+Change resource type from "Bucket" to "Image" for GCP images
+"""
+CHUNK_SIZE = 1000
+LOG = logging.getLogger(__name__)
+
+
+class Migration(BaseMigration):
+    @property
+    def mongo_resources(self):
+        return self.db.resources
+
+    @property
+    def rest_cl(self):
+        if self._rest_cl is None:
+            self._rest_cl = RestClient(
+                url=self.config_cl.restapi_url(),
+                secret=self.config_cl.cluster_secret())
+        return self._rest_cl
+
+    def get_cloud_accs(self):
+        cloud_accounts_ids = set()
+        _, organizations = self.rest_cl.organization_list({
+            "with_connected_accounts": True, "is_demo": False})
+        for org in organizations["organizations"]:
+            _, accounts = self.rest_cl.cloud_account_list(
+                org['id'], type="gcp_cnr")
+            for cloud_account in accounts["cloud_accounts"]:
+                if cloud_account["auto_import"]:
+                    cloud_accounts_ids.add(cloud_account["id"])
+        return cloud_accounts_ids
+
+    def upgrade(self):
+        cloud_accs = self.get_cloud_accs()
+        for i, cloud_acc_id in enumerate(list(cloud_accs)):
+            LOG.info("Starting processing for cloud account %s (%s/%s)" % (
+                cloud_acc_id, i + 1, len(cloud_accs)))
+            self.mongo_resources.update_many({
+                "cloud_account_id": cloud_acc_id,
+                "resource_type": "Bucket",
+                "name": {"$regex": "Storage(.*) Image(.*)"}
+            }, {"$set": {"resource_type": "Image"}})
+
+    def downgrade(self):
+        pass