Skip to content

backend: kv-configs-reverse to read all from kv #632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
khuzema786 merged 1 commit into from
Nov 19, 2024
Merged

backend: kv-configs-reverse to read all from kv #632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 8 additions & 19 deletions lib/mobility-core/src/Kernel/Beam/Functions.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ where

import Data.Aeson
import Data.Default.Class
import qualified Data.HashMap.Strict as HM
import qualified Data.Serialize as Serialize
import Database.Beam hiding (timestamp)
import Database.Beam.MySQL ()
Expand All @@ -53,7 +54,6 @@ import Kernel.Types.Error
import Kernel.Utils.Error.Throwing (throwError)
import Kernel.Utils.Logging (logDebug)
import Sequelize
import System.Random

-- classes for converting from beam types to ttypes and vice versa
class
Expand Down Expand Up @@ -117,25 +117,14 @@ runInMasterDb m = do

setMeshConfig :: (L.MonadFlow m, HasCallStack) => Text -> Maybe Text -> MeshConfig -> m MeshConfig
setMeshConfig modelName mSchema meshConfig' = do
schema <- maybe (L.throwException $ InternalError "Schema not found") pure mSchema
schema <- maybe (L.throwException $ InternalError "Schema not found in setMeshConfig") pure mSchema
let redisStream = if schema == "atlas_driver_offer_bpp" then "driver-db-sync-stream" else "rider-db-sync-stream" -- lets change when we enable for dashboards
tables <- L.getOption KBT.Tables
randomIntV <- L.runIO (randomRIO (1, 100) :: IO Int)
case tables of
Nothing -> L.throwException $ InternalError "Tables not found"
Just tables' -> do
let enableKVForWriteAlso = tables'.enableKVForWriteAlso
enableKVForRead = tables'.enableKVForRead
tableObject = find (\table' -> nameOfTable table' == modelName) enableKVForWriteAlso
updatedMeshConfig <- case tableObject of
Nothing -> pure $ meshConfig' {meshEnabled = False, kvHardKilled = modelName `notElem` enableKVForRead, ecRedisDBStream = redisStream}
Just table' -> do
let redisTtl' = fromMaybe (meshConfig'.redisTtl) (table'.redisTtl)
if fromIntegral (percentEnable table') >= randomIntV
then pure $ meshConfig' {meshEnabled = True, kvHardKilled = modelName `notElem` enableKVForRead, ecRedisDBStream = redisStream, redisTtl = redisTtl'}
else pure $ meshConfig' {meshEnabled = False, kvHardKilled = modelName `notElem` enableKVForRead, ecRedisDBStream = redisStream}
L.logDebug ("setMeshConfig" :: Text) $ "meshConfig for table: " <> modelName <> " : " <> show updatedMeshConfig
pure updatedMeshConfig
tables' <- L.getOption KBT.Tables >>= maybe (L.throwException $ InternalError "Tables not found in setMeshConfig") pure
if modelName `elem` tables'.disableForKV
then pure $ meshConfig' {ecRedisDBStream = redisStream}
else do
let redisTtl' = HM.lookupDefault meshConfig'.redisTtl modelName tables'.kvTablesTtl
pure $ meshConfig' {meshEnabled = True, kvHardKilled = False, ecRedisDBStream = redisStream, redisTtl = redisTtl'}

withUpdatedMeshConfig :: forall table m a. (L.MonadFlow m, HasCallStack, ModelMeta table) => Proxy table -> (MeshConfig -> m a) -> m a
withUpdatedMeshConfig _ mkAction = do
Expand Down
24 changes: 4 additions & 20 deletions lib/mobility-core/src/Kernel/Types/Common.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import Data.Aeson
import Data.ByteString.Internal (ByteString)
import Data.Fixed (Centi, Fixed (MkFixed))
import Data.Generics.Labels ()
import qualified Data.HashMap.Strict as HM
import Data.OpenApi hiding (value)
import Data.Text as T
import qualified Data.Vector as V
Expand Down Expand Up @@ -57,34 +58,17 @@ import Kernel.Types.Logging as Common
import Kernel.Types.MonadGuid as Common
import Kernel.Types.Price as Common
import Kernel.Types.Time as Common
import Kernel.Utils.Dhall (FromDhall, Natural)
import Kernel.Utils.Dhall (FromDhall)

newtype IdObject = IdObject
{ id :: Text
}
deriving stock (Generic, Show)
deriving anyclass (ToJSON, FromJSON, ToSchema)

data KVTable = KVTable
{ nameOfTable :: Text,
percentEnable :: Natural,
redisTtl :: Maybe Integer
}
deriving (Generic, Eq, Show, ToJSON, FromJSON, FromDhall, Ord)

instance HasSqlValueSyntax be String => HasSqlValueSyntax be KVTable where
sqlValueSyntax = autoSqlValueSyntax

instance BeamSqlBackend be => B.HasSqlEqualityCheck be KVTable

instance FromBackendRow Postgres KVTable

instance FromField KVTable where
fromField = fromFieldJSON

data Tables = Tables
{ enableKVForWriteAlso :: [KVTable],
enableKVForRead :: [Text],
{ disableForKV :: [Text],
kvTablesTtl :: HM.HashMap Text Integer,
useCAC :: [Text],
useCACForFrontend :: Bool,
readFromMasterDb :: [Text]
Copy link
Contributor

@khuzema786 khuzema786 Nov 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

{
  "enableKVForRead": ["table1", "table2", "table3", ...],
  "enableKVForWriteAlso": [{ "nameOfTable" : "table1", "percentEnable" : 100 }, ...],
  "disableForKV": ["table1", "table2", "table3", ...],
  "kvTablesTtl": {
    "table1": 3600,
    "table2": 7200,
    "table3": 14400,
     ...
  },
  "useCAC": ["table4", "table5", ...],
  "useCACForFrontend": true,
  "readFromMasterDb": ["table6", "table7", ...]
}

Expand Down