From cbe81b60b4e56994208a0a0eb6f7d66b0334a098 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 28 Feb 2020 17:05:28 +0300 Subject: [PATCH 001/119] add new actors --- src/main/scala/encry/network/DM.scala | 7 +++++++ src/main/scala/encry/network/MessageBuilder.scala | 7 +++++++ src/main/scala/encry/network/NetworkRouter.scala | 7 +++++++ src/main/scala/encry/network/PK.scala | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100644 src/main/scala/encry/network/DM.scala create mode 100644 src/main/scala/encry/network/MessageBuilder.scala create mode 100644 src/main/scala/encry/network/NetworkRouter.scala create mode 100644 src/main/scala/encry/network/PK.scala diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala new file mode 100644 index 0000000000..ea5b777bf2 --- /dev/null +++ b/src/main/scala/encry/network/DM.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class DM extends Actor{ + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala new file mode 100644 index 0000000000..1dd6b4ea0c --- /dev/null +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class MessageBuilder extends Actor { + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala new file mode 100644 index 0000000000..5885fd127e --- /dev/null +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class NetworkRouter extends Actor { + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala new file mode 100644 index 0000000000..835cdb926e --- /dev/null +++ b/src/main/scala/encry/network/PK.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class PK extends Actor{ + override def receive: Receive = ??? +} From 36f4f7d97c780722f379f001ae8d62f7ef4e9b95 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 2 Mar 2020 17:23:47 +0300 Subject: [PATCH 002/119] add some messages --- src/main/scala/encry/network/BlackList.scala | 9 +- src/main/scala/encry/network/DM.scala | 8 +- .../scala/encry/network/MessageBuilder.scala | 31 ++++- src/main/scala/encry/network/Messages.scala | 16 +++ .../encry/network/NetworkController.scala | 1 - .../scala/encry/network/NetworkRouter.scala | 112 +++++++++++++++++- .../encry/network/NodeViewSynchronizer.scala | 3 - src/main/scala/encry/network/PK.scala | 28 ++++- .../scala/encry/view/NodeViewHolder.scala | 1 + 9 files changed, 191 insertions(+), 18 deletions(-) create mode 100644 src/main/scala/encry/network/Messages.scala diff --git a/src/main/scala/encry/network/BlackList.scala b/src/main/scala/encry/network/BlackList.scala index 9cbb5ea8ff..a360757969 100644 --- a/src/main/scala/encry/network/BlackList.scala +++ b/src/main/scala/encry/network/BlackList.scala @@ -1,11 +1,12 @@ package encry.network import java.net.InetAddress + import encry.network.BlackList.BanType.{PermanentBan, TemporaryBan} import encry.network.BlackList._ -import encry.settings.EncryAppSettings +import encry.settings.BlackListSettings -final case class BlackList(settings: EncryAppSettings, +final case class BlackList(settings: BlackListSettings, private val blackList: Map[InetAddress, (BanReason, BanTime, BanType)]) { def contains(peer: InetAddress): Boolean = blackList.contains(peer) @@ -16,7 +17,7 @@ final case class BlackList(settings: EncryAppSettings, }))) def cleanupBlackList: BlackList = BlackList(settings, blackList.filterNot { case (_, (_, banTime, banType)) => - banType != PermanentBan && (System.currentTimeMillis() - banTime.time >= settings.blackList.banTime.toMillis) + banType != PermanentBan && (System.currentTimeMillis() - banTime.time >= settings.banTime.toMillis) }) def remove(peer: InetAddress): BlackList = BlackList(settings, blackList - peer) @@ -59,6 +60,6 @@ object BlackList { final case class BanTime(time: Long) extends AnyVal - def apply(settings: EncryAppSettings): BlackList = + def apply(settings: BlackListSettings): BlackList = BlackList(settings, Map.empty[InetAddress, (BanReason, BanTime, BanType)]) } \ No newline at end of file diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index ea5b777bf2..2133fe0742 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,7 +1,11 @@ package encry.network -import akka.actor.Actor +import akka.actor.{Actor, Props} -class DM extends Actor{ +class DM extends Actor { override def receive: Receive = ??? } + +object DM { + def props(): Props = ??? +} diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 1dd6b4ea0c..5efffac15a 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -1,7 +1,32 @@ package encry.network -import akka.actor.Actor +import akka.actor.{Actor, ActorRef, Props} +import akka.pattern._ +import akka.util.Timeout +import encry.network.MessageBuilder.GetPeers +import encry.network.Messages.MessageToNetwork +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, SendSyncInfo} -class MessageBuilder extends Actor { - override def receive: Receive = ??? +import scala.concurrent.duration._ + +case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { + + import context.dispatcher + + implicit val timeout: Timeout = Timeout(10 seconds) + + override def receive: Receive = { + case RequestFromLocal(peer, modTypeId, modsIds) => + case SendSyncInfo(syncInfo) => + (peersKeeper ? GetPeers).mapTo[List[ActorRef]].map { peers => + peers.foreach(_ ! syncInfo) + } + } } + +object MessageBuilder { + + case object GetPeers + + def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) +} \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala new file mode 100644 index 0000000000..193e887081 --- /dev/null +++ b/src/main/scala/encry/network/Messages.scala @@ -0,0 +1,16 @@ +package encry.network + +import encry.network.PeerConnectionHandler.ConnectedPeer +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + +object Messages { + + sealed trait MessageToNetwork + object MessageToNetwork { + final case class RequestFromLocal(source: ConnectedPeer, + modifierTypeId: ModifierTypeId, + modifierIds: Seq[ModifierId]) extends MessageToNetwork + final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork + } +} diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 983f5daf53..af2d242cac 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -15,7 +15,6 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeersKeeper._ import encry.settings.{EncryAppSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage - import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.language.{existentials, postfixOps} diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 5885fd127e..a1ba21cf58 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -1,7 +1,113 @@ package encry.network -import akka.actor.Actor +import java.net.InetSocketAddress -class NetworkRouter extends Actor { - override def receive: Receive = ??? +import akka.actor.{Actor, ActorRef, Props} +import akka.io.Tcp.{Bind, Bound, CommandFailed, Connect, Connected} +import akka.io.{IO, Tcp} +import akka.io.Tcp.SO.KeepAlive +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.InvalidNetworkMessage +import encry.network.Messages.MessageToNetwork +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} +import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, OutgoingConnectionFailed, PeerForConnection, VerifyConnection} +import encry.settings.NetworkSettings +import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage + +class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging { + + import context.system + + var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty + + IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) + + val peersKeeper = context.system.actorOf(PK.props(), "peersKeeper") + val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") + val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress + + override def receive: Receive = bindingLogic orElse businessLogic orElse { + case RegisterMessagesHandler(types, handler) => + logger.info(s"Registering handlers for ${types.mkString(",")}.") + val ids = types.map(_._1) + messagesHandlers += (ids -> handler) + case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") + case msg => logger.warn(s"NetworkController: got something strange $msg.") + } + + def bindingLogic: Receive = { + case Bound(address) => + logger.info(s"Successfully bound to the port ${address.getPort}.") + case CommandFailed(add: Bind) => + logger.info(s"Node can't be bind to the address: ${add.localAddress}.") + context.stop(self) + } + + def businessLogic: Receive = { + case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => + logger.debug(s"Got ${message.messageName} on the NetworkController.") + findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) + case MessageFromNetwork(message, Some(remote)) => + peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) + logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg), "peersKeeper") + } + + def peersLogic: Receive = { + case PeerForConnection(peer) => + logger.info(s"Network router got new peer for connection: $peer. Trying to set connection with remote...") + IO(Tcp) ! Connect( + peer, + None, + KeepAlive(true) :: Nil, + Some(settings.connectionTimeout), + pullMode = true + ) + + case Connected(remote, localAddress) => + logger.info(s"Network router got 'Connected' message from: $remote. " + + s"Trying to set stable connection with remote... " + + s"Local TCP endpoint is: $localAddress.") + peersKeeper ! VerifyConnection(remote, sender()) + + case ConnectionVerified(remote, remoteConnection, connectionType) => + logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") + val peerConnectionHandler: ActorRef = context.actorOf( + PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, settings) + .withDispatcher("network-dispatcher") + ) + peerConnectionHandler ! StartInteraction + + case HandshakedDone(remote) => + logger.info(s"Network controller got approvement from peer handler about successful handshake. " + + s"Sending to peerKeeper connected peer.") + peersKeeper ! HandshakedDone(remote) + + case ConnectionStopped(peer) => + logger.info(s"Network controller got signal about breaking connection with: $peer. " + + s"Sending to peerKeeper actual information.") + peersKeeper ! ConnectionStopped(peer) + + case CommandFailed(connect: Connect) => + logger.info(s"Failed to connect to: ${connect.remoteAddress}.") + peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) + } + + private def findHandler(message: NetworkMessage, + messageId: Byte, + remote: ConnectedPeer, + mH: Map[Seq[Byte], ActorRef]): Unit = + mH.find(_._1.contains(messageId)).map(_._2) match { + case Some(handler) => + handler ! DataFromPeer(message, remote) + logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") + case None => logger.info("No handlers found for message: " + message.messageName) + } +} + +object NetworkRouter { + + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index db92cc82ad..931768507c 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -268,9 +268,6 @@ object NodeViewSynchronizer { status: encry.consensus.HistoryConsensus.HistoryComparisonResult, extension: Option[Seq[(ModifierTypeId, ModifierId)]]) - final case class RequestFromLocal(source: ConnectedPeer, - modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId]) sealed trait CLIPeer final case class PeerFromCli(address: InetSocketAddress) extends CLIPeer diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 835cdb926e..7adda9e5db 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -1,7 +1,31 @@ package encry.network -import akka.actor.Actor +import java.net.{InetAddress, InetSocketAddress} + +import akka.actor.{Actor, Props} +import encry.settings.{BlackListSettings, NetworkSettings} + +import scala.util.Try + +class PK(networkSettings: NetworkSettings, + blacklistSettings: BlackListSettings) extends Actor { + + var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() + + var blackList: BlackList = BlackList(blacklistSettings) + + var knownPeers: Set[InetAddress] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet -class PK extends Actor{ override def receive: Receive = ??? + + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || + networkSettings.declaredAddress.contains(address) || + InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || + InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) +} + +object PK { + def props(networkSettings: NetworkSettings, + blacklistSettings: BlackListSettings): Props = Props(new PK(networkSettings, blacklistSettings)) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index eed63d2e9a..9099a4dc59 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -13,6 +13,7 @@ import encry.api.http.DataHolderForApi import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings From 2395da36128d0daba095e1a6209f990c9c3b8465 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 17:25:02 +0300 Subject: [PATCH 003/119] nvh group frame added --- src/main/resources/application.conf | 9 +++++- .../scala/encry/nvg/IntermediaryNVH.scala | 30 +++++++++++++++++++ .../scala/encry/nvg/ModifiersValidator.scala | 20 +++++++++++++ .../encry/nvg/NetworkMessagesProcessor.scala | 11 +++++++ src/main/scala/encry/nvg/NodeViewHolder.scala | 11 +++++++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/main/scala/encry/nvg/IntermediaryNVH.scala create mode 100644 src/main/scala/encry/nvg/ModifiersValidator.scala create mode 100644 src/main/scala/encry/nvg/NetworkMessagesProcessor.scala create mode 100644 src/main/scala/encry/nvg/NodeViewHolder.scala diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 6718d3b58d..2c97730e8b 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -151,7 +151,14 @@ akka.http { akka.http.routing { verbose-error-messages = on } - +modifiers-validator-router-dispatcher { + executor = "thread-pool-executor" + # allocate exactly 5 threads for this pool + thread-pool-executor { + core-pool-size-min = 5 + core-pool-size-max = 5 + } +} mining-dispatcher { type = Dispatcher executor = "thread-pool-executor" diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala new file mode 100644 index 0000000000..b8292741de --- /dev/null +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -0,0 +1,30 @@ +package encry.nvg + +import akka.actor.{ Actor, ActorRef } +import akka.routing.BalancingPool +import com.typesafe.scalalogging.StrictLogging +import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings + +class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { + + val networkMessagesProcessor: ActorRef = + context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") + val nodeViewHolder: ActorRef = + context.actorOf(NodeViewHolder.props, name = "Node-view-holder") + val modifiersValidatorRouter: ActorRef = + context.actorOf( + BalancingPool(5) + .props(ModifiersValidator.props(nodeViewHolder, settings)), + name = "Modifiers-validator-router" + ) + + override def receive: Receive = { + case msg @ ModifiersForValidating(_, _, _) => modifiersValidatorRouter ! msg + case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case _ => + } +} + +object IntermediaryNVH {} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala new file mode 100644 index 0000000000..a4c57c36e3 --- /dev/null +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -0,0 +1,20 @@ +package encry.nvg + +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage + +class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { + override def receive: Receive = { + case ModifiersForValidating(remote, typeId, filteredModifiers) => + case _ => + } +} + +object ModifiersValidator { + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = + Props(new ModifiersValidator(nodeViewHolderRef, settings)) +} diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala new file mode 100644 index 0000000000..6cb0c78b59 --- /dev/null +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -0,0 +1,11 @@ +package encry.nvg + +import akka.actor.{ Actor, Props } + +class NetworkMessagesProcessor extends Actor { + override def receive: Receive = ??? +} + +object NetworkMessagesProcessor { + def props: Props = Props(new NetworkMessagesProcessor) +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala new file mode 100644 index 0000000000..2aaed78db1 --- /dev/null +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -0,0 +1,11 @@ +package encry.nvg + +import akka.actor.{ Actor, Props } + +class NodeViewHolder extends Actor { + override def receive: Receive = ??? +} + +object NodeViewHolder { + def props: Props = Props(new NodeViewHolder) +} From 45f7662fc275f6bebce45d5f03c89bc3c6c47cff Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 18:19:13 +0300 Subject: [PATCH 004/119] validator frame added --- .../scala/encry/nvg/IntermediaryNVH.scala | 20 ++++- .../scala/encry/nvg/ModifiersValidator.scala | 90 +++++++++++++++++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 6 +- .../encry/view/history/HistoryReader.scala | 17 ++++ 4 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/main/scala/encry/view/history/HistoryReader.scala diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index b8292741de..994fc943dd 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -5,7 +5,11 @@ import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.network.DownloadedModifiersValidator.ModifiersForValidating import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.view.history.HistoryReader +import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { @@ -20,10 +24,20 @@ class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLoggi name = "Modifiers-validator-router" ) + var historyReader: HistoryReader = HistoryReader.empty + override def receive: Receive = { - case msg @ ModifiersForValidating(_, _, _) => modifiersValidatorRouter ! msg - case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg - case _ => + case ModifiersForValidating(remote, typeId, modifiers) => + logger.info(s"Got ${modifiers.size} modifiers of type $typeId for validation.") + modifiers.foreach { + case (id: ModifierId, bytes: Array[Byte]) => + modifiersValidatorRouter ! ModifierForValidation(historyReader, id, typeId, bytes, remote) + } + case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case UpdateHistoryReader(newReader: HistoryReader) => + historyReader = newReader + networkMessagesProcessor ! newReader + case _ => } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index a4c57c36e3..68861400c7 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -1,20 +1,98 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import HeaderProto.HeaderProtoMessage +import PayloadProto.PayloadProtoMessage +import akka.actor.{ Actor, ActorRef, Props } +import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging -import encry.network.DownloadedModifiersValidator.ModifiersForValidating -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.modifiers.history.{ HeaderUtils, PayloadUtils } +import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidationException } +import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.ModifierForValidation import encry.settings.EncryAppSettings -import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage +import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + +import scala.util.{ Failure, Try } class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { override def receive: Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - case _ => + case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => + fromProto(modifierTypeId, modifierBytes) match { + case Left(error) => + case Right(modifier) => + val preSemanticValidation: Either[PreSemanticValidationException, Unit] = + isPreSemanticValid(modifier, reader, settings) + val syntacticValidation: Boolean = + isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) + if (preSemanticValidation.isRight && syntacticValidation) { + logger.debug(s"Modifier ${modifier.encodedId} is valid.") + nodeViewHolderRef ! ModifierFromRemote(modifier) + } else if (!syntacticValidation) { + logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") + context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + } else { + preSemanticValidation.leftMap { + case IllegalHeight(error) => + logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") + context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) + } + } + } + } + + private def isPreSemanticValid( + modifier: PersistentModifier, + historyReader: HistoryReader, + settings: EncryAppSettings + ): Either[PreSemanticValidationException, Unit] = + modifier match { + case header: Header => + val bestHeaderHeight: Int = historyReader.getBestHeaderHeight + Either.cond( + bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height, + (), + IllegalHeight( + s"Height of received header is ${header.height}. Current best header height is $bestHeaderHeight. " + + s"Max possible received header's height is ${bestHeaderHeight - settings.constants.MaxRollbackDepth}." + ) + ) + case _: Payload => ().asRight[PreSemanticValidationException] + } + + private def fromProto( + modType: ModifierTypeId, + bytes: Array[Byte] + ): Either[Throwable, PersistentModifier] = + Either.fromTry(modType match { + case Header.modifierTypeId => HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes)) + case Payload.modifierTypeId => PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes)) + }) + + def isSyntacticallyValid( + modifier: PersistentModifier, + modifierIdSize: Int + ): Boolean = modifier match { + case h: Header => HeaderUtils.syntacticallyValidity(h, modifierIdSize).isSuccess + case p: Payload => PayloadUtils.syntacticallyValidity(p, modifierIdSize).isSuccess } } object ModifiersValidator { + + final case class ModifierForValidation( + historyReader: HistoryReader, + modifierId: ModifierId, + modifierTypeId: ModifierTypeId, + modifierBytes: Array[Byte], + remote: ConnectedPeer + ) + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 2aaed78db1..4ad1757d18 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,11 +1,15 @@ package encry.nvg -import akka.actor.{ Actor, Props } +import akka.actor.{Actor, Props} +import encry.view.history.{History, HistoryReader} class NodeViewHolder extends Actor { override def receive: Receive = ??? } object NodeViewHolder { + + final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal + def props: Props = Props(new NodeViewHolder) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala new file mode 100644 index 0000000000..7388d4af38 --- /dev/null +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -0,0 +1,17 @@ +package encry.view.history + +trait HistoryReader { + + def getBestHeaderHeight: Int + +} + +object HistoryReader { + def empty: HistoryReader = new HistoryReader { + def getBestHeaderHeight = 0 + } + + def apply(): HistoryReader = new HistoryReader { + def getBestHeaderHeight = 1 + } +} \ No newline at end of file From 17a01225fda736e57e9306ad49b6ad91f8adcd9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 18:27:39 +0300 Subject: [PATCH 005/119] validator frame extended --- .../DownloadedModifiersValidator.scala | 35 ------------------- .../scala/encry/nvg/ModifiersValidator.scala | 18 +++++++--- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 74f79146fb..bfeb7c691b 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -37,41 +37,6 @@ class DownloadedModifiersValidator(modifierIdSize: Int, } def workingCycle(history: History): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) if typeId != Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - ModifiersToNetworkUtils.fromProto(typeId, bytes) match { - case Success(modifier) => - val syntacticValidation: Boolean = ModifiersToNetworkUtils.isSyntacticallyValid(modifier, modifierIdSize) - val preSemanticValidation: Either[HeaderUtils.PreSemanticValidationException, Unit] = - ModifiersToNetworkUtils.isPreSemanticValidation(modifier, history, settings) - if (syntacticValidation && preSemanticValidation.isRight) { - logger.debug( - s"Modifier: ${modifier.encodedId} after testApplicable is correct. " + - s"Sending validated modifier to NodeViewHolder" - ) - influxRef.foreach(_ ! ValidatedModifierFromNetwork(typeId)) - nodeViewHolder ! ModifierFromRemote(modifier) - } else { - logger.info( - s"Modifier with id: ${modifier.encodedId} of type: $typeId invalid cause of:" + - s"isSyntacticallyValid = false or $preSemanticValidation" - ) - if (!syntacticValidation) peersKeeper ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - else - preSemanticValidation match { - case Left(value) => peersKeeper ! BanPeer(remote, PreSemanticInvalidModifier(value.error)) - case Right(_) => - } - nodeViewSync ! InvalidModifier(id) - } - case Failure(ex) => - peersKeeper ! BanPeer(remote, CorruptedSerializedBytes) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - nodeViewSync ! InvalidModifier(id) - } - } - case ModifiersForValidating(remote, typeId, filteredModifiers) => typeId match { case Transaction.modifierTypeId => diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 68861400c7..f16346b79d 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -5,9 +5,14 @@ import PayloadProto.PayloadProtoMessage import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging -import encry.modifiers.history.{ HeaderUtils, PayloadUtils } import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidationException } -import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } +import encry.modifiers.history.{ HeaderUtils, PayloadUtils } +import encry.network.BlackList.BanReason.{ + CorruptedSerializedBytes, + PreSemanticInvalidModifier, + SyntacticallyInvalidPersistentModifier +} +import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -16,15 +21,18 @@ import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -import scala.util.{ Failure, Try } - class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { + override def receive: Receive = { case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => fromProto(modifierTypeId, modifierBytes) match { case Left(error) => + logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") + context.parent ! BanPeer(remote, CorruptedSerializedBytes) + context.parent ! InvalidModifier(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -36,11 +44,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + context.parent ! InvalidModifier(modifierId) } else { preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) + context.parent ! InvalidModifier(modifierId) } } } From 12a0f5c53e2154cc85362cbf1c8d05b7d4e6a465 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 14:35:24 +0300 Subject: [PATCH 006/119] added inner impl of nvh --- .../scala/encry/nvg/IntermediaryNVH.scala | 9 +- .../scala/encry/nvg/ModifiersValidator.scala | 11 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 462 +++++++++++++++++- .../scala/encry/view/NodeViewHolder.scala | 324 ------------ 4 files changed, 467 insertions(+), 339 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 994fc943dd..f1af14f4b0 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -3,15 +3,17 @@ package encry.nvg import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging -import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId -class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { +class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) extends Actor with StrictLogging { val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") @@ -37,7 +39,8 @@ class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLoggi case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case _ => + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index f16346b79d..512620a119 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -45,14 +45,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) context.parent ! InvalidModifier(modifierId) - } else { + } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) context.parent ! InvalidModifier(modifierId) } - } } } @@ -64,14 +63,12 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings modifier match { case header: Header => val bestHeaderHeight: Int = historyReader.getBestHeaderHeight - Either.cond( - bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height, - (), + if (bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height) ().asRight + else IllegalHeight( s"Height of received header is ${header.height}. Current best header height is $bestHeaderHeight. " + s"Max possible received header's height is ${bestHeaderHeight - settings.constants.MaxRollbackDepth}." - ) - ) + ).asLeft case _: Payload => ().asRight[PreSemanticValidationException] } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 4ad1757d18..f83295b81d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,15 +1,467 @@ package encry.nvg -import akka.actor.{Actor, Props} -import encry.view.history.{History, HistoryReader} +import java.io.File -class NodeViewHolder extends Actor { - override def receive: Receive = ??? +import akka.actor.{ Actor, ActorRef, Props } +import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp +import encry.EncryApp.system +import encry.api.http.DataHolderForApi +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + ChangedHistory, + ChangedState, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} +import encry.nvg.NodeViewHolder.NodeView +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.{ + BestHeaderInChain, + EndOfApplyingModifier, + HeightStatistics, + ModifierAppendedToHistory, + ModifierAppendedToState, + StartApplyingModifier, + StateUpdating, + TransactionsInBlock +} +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.NetworkTimeProvider +import encry.view.ModifiersCache +import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError +import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } +import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId +import encry.view.fast.sync.SnapshotHolder.{ + HeaderChainIsSynced, + RequiredManifestHeightAndId, + SnapshotChunk, + TreeChunks +} +import encry.view.history.{ History, HistoryReader } +import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.view.state.UtxoState +import encry.view.state.avlTree.AvlTree +import encry.view.wallet.EncryWallet +import io.iohk.iodb.ByteArrayWrapper +import org.apache.commons.io.FileUtils +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } + +import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.concurrent.ExecutionContextExecutor +import scala.util.{ Failure, Success, Try } + +class NodeViewHolder( + settings: EncryAppSettings, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging + with AutoCloseable { + + import context.dispatcher + + var nodeView: NodeView = restoreState().getOrElse(genesisState) + + var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + + override def receive: Receive = { + case ModifierFromRemote(modifier) => + } + + //todo refactor loop + def computeApplications(): Unit = { + val mods = ModifiersCache.popCandidate(nodeView.history) + if (mods.nonEmpty) { + logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") + mods.foreach(mod => pmodModify(mod)) + computeApplications() + } else Unit + } + + //todo replace with reader + def updateNodeView( + updatedHistory: Option[History] = None, + updatedState: Option[UtxoState] = None, + updatedVault: Option[EncryWallet] = None + ): Unit = { + val newNodeView: NodeView = NodeView( + updatedHistory.getOrElse(nodeView.history), + updatedState.getOrElse(nodeView.state), + updatedVault.getOrElse(nodeView.wallet) + ) + if (updatedHistory.nonEmpty) context.parent ! ChangedHistory(newNodeView.history) + if (updatedState.nonEmpty) context.parent ! ChangedState(newNodeView.state) + nodeView = newNodeView + } + + def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = + pi.toDownload.foreach { + case (tid, id) => + if (tid != Transaction.modifierTypeId) + logger.info( + s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + + s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}." + ) + if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) + system.actorSelection("/user/nodeViewSynchronizer") ! DownloadRequest(tid, id, previousModifier) + else + logger.info( + s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false" + ) + } + + def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], + rollbackPoint: ModifierId): IndexedSeq[PersistentModifier] = { + val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) + if (idx == -1) IndexedSeq() else suffix.drop(idx) + } + + @scala.annotation.tailrec + private def updateState(history: History, + state: UtxoState, + progressInfo: ProgressInfo, + suffixApplied: IndexedSeq[PersistentModifier], + isLocallyGenerated: Boolean = false): (History, UtxoState, Seq[PersistentModifier]) = { + logger.info(s"\nStarting updating state in updateState function!") + if (!isLocallyGenerated) progressInfo.toApply.foreach { + case header: Header => requestDownloads(progressInfo, Some(header.id)) + case _ => requestDownloads(progressInfo, None) + } + val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) + val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = + if (progressInfo.chainSwitchingNeeded) { + branchingPointOpt.map { branchPoint => + if (!state.version.sameElements(branchPoint)) { + val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + context.system.actorSelection("/user/miner") ! DisableMining + state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, + ModifierId !@@ branchPoint) + } else Success(state) -> IndexedSeq() + }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) + } else Success(state) -> suffixApplied + stateToApplyTry match { + case Success(stateToApply) => + context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) + val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) + val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { + case (u, modToApply) => + val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 + if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { + case Right(stateAfterApply) => + influxRef.foreach( + ref => + modToApply match { + case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) + case _ => + } + ) + val newHis: History = history.reportModifierIsValid(modToApply) + //dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) + modToApply match { + case header: Header => + val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth + if (requiredHeight % settings.constants.SnapshotCreationHeight == 0) { + newHis.lastAvailableManifestHeight = requiredHeight + logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") + } + case _ => + } + newHis.getHeaderOfBestBlock.foreach { header: Header => + val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) + val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) + val isCorrectCreationHeight: Boolean = + header.height % settings.constants.SnapshotCreationHeight == 0 + val isGenesisHeader: Boolean = header.height == settings.constants.GenesisHeight + if (settings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && + !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { + val startTime = System.currentTimeMillis() + logger.info(s"Start chunks creation for new snapshot") + import encry.view.state.avlTree.utils.implicits.Instances._ + val chunks: List[SnapshotChunk] = + AvlTree.getChunks( + stateAfterApply.tree.rootNode, + currentChunkHeight = settings.snapshotSettings.chunkDepth, + stateAfterApply.tree.avlStorage + ) + system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) + potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds + logger.info( + s"State tree successfully processed for snapshot. " + + s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) + } + } + if (settings.node.mining && progressInfo.chainSwitchingNeeded) + context.system.actorSelection("/user/miner") ! StartMining + context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + if (newHis.getBestHeaderId.exists( + bestHeaderId => + newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) + )) newHis.isFullChainSynced = true + influxRef.foreach { ref => + logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") + ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) + val isBlock: Boolean = modToApply match { + case _: Block => true + case _: Payload => true + case _ => false + } + if (isBlock) ref ! ModifierAppendedToState(success = true) + } + UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) + case Left(e) => + logger.info(s"Application to state failed cause $e") + val (newHis: History, newProgressInfo: ProgressInfo) = + history.reportModifierIsInvalid(modToApply) + context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) + UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) + } else u + } + uf.failedMod match { + case Some(_) => + uf.history.updateIdsForSyncInfo() + updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) + case None => (uf.history, uf.state, uf.suffix) + } + case Failure(e) => + context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) + EncryApp.forceStopApplication(500, s"Rollback failed: $e") + } + } + + def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + if (!nodeView.history.isModifierDefined(pmod.id)) { + logger.debug( + s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + ) + val startAppHistory = System.currentTimeMillis() + if (settings.influxDB.isDefined) + context.system + .actorSelection("user/statsSender") ! + StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) + nodeView.history.append(pmod) match { + case Right((historyBeforeStUpdate, progressInfo)) => + logger.info( + s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + ) + logger.debug( + s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}" + ) + if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() + influxRef.foreach { ref => + ref ! EndOfApplyingModifier(pmod.id) + val isHeader: Boolean = pmod match { + case _: Header => true + case _: Payload => false + } + ref ! ModifierAppendedToHistory(isHeader, success = true) + } + if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && + historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { + logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") + logger.info( + s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" + ) + historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { + h => + system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( + historyBeforeStUpdate.lastAvailableManifestHeight, + Algos.hash(h.stateRoot ++ h.id) + ) + } + } + logger.info( + s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo" + ) + if (progressInfo.toApply.nonEmpty) { + logger.info( + s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}" + ) + val startPoint: Long = System.currentTimeMillis() + val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = + updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) + if (newHistory.isHeadersChainSynced) + system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced + if (settings.influxDB.isDefined) + context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) + sendUpdatedInfoToMemoryPool(progressInfo.toRemove) + if (progressInfo.chainSwitchingNeeded) + nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get + blocksApplied.foreach(nodeView.wallet.scanPersistent) + logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") + if (settings.influxDB.isDefined) + newHistory.getBestHeader.foreach( + header => context.actorSelection("/user/statsSender") ! BestHeaderInChain(header) + ) + if (newHistory.isFullChainSynced) { + logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") + ModifiersCache.setChainSynced() + system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced + system.actorSelection("/user/miner") ! FullBlockChainIsSynced + } + updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) + } else { + influxRef.foreach { ref => + logger.info( + s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}" + ) + ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) + } + if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) + context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) + logger.info(s"\nProgress info is empty") + updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) + } + case Left(e) => + logger.debug( + s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + + s" to history caused $e" + ) + context.system.eventStream + .publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) + } + } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") + + def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + + def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { + val rolledBackTxs: IndexedSeq[Transaction] = toRemove + .flatMap(extractTransactions) + .toIndexedSeq + //if (rolledBackTxs.nonEmpty) memoryPoolRef ! RolledBackTransactions(rolledBackTxs) + } + + def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { + case b: Block => b.payload.txs + case p: Payload => p.txs + case _ => Seq.empty[Transaction] + } + + def genesisState: NodeView = { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdir() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") + val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) + val history: History = History.readOrGenerate(settings, timeProvider) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + NodeView(history, state, wallet) + } + + def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = + if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) + try { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val history: History = History.readOrGenerate(settings, timeProvider) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + history, + influxRef + ) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + Some(NodeView(history, state, wallet)) + } catch { + case ex: Throwable => + logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") + new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) + Some(genesisState) + } else { + None + } + + def getRecreatedState( + version: Option[VersionTag] = None, + digest: Option[ADDigest] = None, + influxRef: Option[ActorRef] + ): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + UtxoState.create(stateDir, rootsDir, settings, influxRef) + } + + def restoreConsistentState( + stateIn: UtxoState, + history: History, + influxRefActor: Option[ActorRef] + ): UtxoState = + (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { + case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => + logger.info(s"State and history are both empty on startup") + stateIn + case (_, None, _, _) => + logger.info( + s"State and history are inconsistent." + + s" History is empty on startup, rollback state to genesis." + ) + getRecreatedState(influxRef = influxRefActor) + case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => + val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) + val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) + logger.info( + s"State and history are inconsistent." + + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + + s"apply ${newChain.length} modifiers" + ) + val additionalBlocks = + (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") + rollbackId + .map(_ => state.restore(additionalBlocks).get) + .getOrElse(getRecreatedState(influxRef = influxRefActor)) + } + + override def close(): Unit = { + nodeView.history.close() + nodeView.state.close() + nodeView.wallet.close() + } } object NodeViewHolder { final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal - def props: Props = Props(new NodeViewHolder) + final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) + + def props( + settings: EncryAppSettings, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] + ): Props = Props(new NodeViewHolder(settings, timeProvider, influxRef)) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9099a4dc59..404f9a52ae 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -51,9 +51,6 @@ class NodeViewHolder(memoryPoolRef: ActorRef, implicit val exCon: ExecutionContextExecutor = context.dispatcher - case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) - - var nodeView: NodeView = restoreState().getOrElse(genesisState(influxRef)) context.system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(nodeView.history) dataHolder ! UpdatedHistory(nodeView.history) @@ -167,327 +164,6 @@ class NodeViewHolder(memoryPoolRef: ActorRef, case msg => logger.error(s"Got strange message on nvh: $msg") } - //todo refactor loop - def computeApplications(): Unit = { - val mods = ModifiersCache.popCandidate(nodeView.history) - if (mods.nonEmpty) { - logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - mods.foreach(mod => pmodModify(mod)) - computeApplications() - } - else Unit - } - - def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - - def updateNodeView(updatedHistory: Option[History] = None, - updatedState: Option[UtxoState] = None, - updatedVault: Option[EncryWallet] = None): Unit = { - val newNodeView: NodeView = NodeView(updatedHistory.getOrElse(nodeView.history), - updatedState.getOrElse(nodeView.state), - updatedVault.getOrElse(nodeView.wallet)) - if (updatedHistory.nonEmpty) { - system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(newNodeView.history) - context.system.eventStream.publish(ChangedHistory(newNodeView.history)) - } - if (updatedState.nonEmpty) context.system.eventStream.publish(ChangedState(newNodeView.state)) - nodeView = newNodeView - } - - def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = - pi.toDownload.foreach { case (tid, id) => - if (tid != Transaction.modifierTypeId) logger.info(s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + - s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}.") - if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) - system.actorSelection("/user/nodeViewSynchronizer")! DownloadRequest(tid, id, previousModifier) - else logger.info(s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false") - } - - def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], rollbackPoint: ModifierId): - IndexedSeq[PersistentModifier] = { - val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) - if (idx == -1) IndexedSeq() else suffix.drop(idx) - } - - @scala.annotation.tailrec - private def updateState(history: History, - state: UtxoState, - progressInfo: ProgressInfo, - suffixApplied: IndexedSeq[PersistentModifier], - isLocallyGenerated: Boolean = false): - (History, UtxoState, Seq[PersistentModifier]) = { - logger.info(s"\nStarting updating state in updateState function!") - if (!isLocallyGenerated) progressInfo.toApply.foreach { - case header: Header => requestDownloads(progressInfo, Some(header.id)) - case _ => requestDownloads(progressInfo, None) - } - val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) - val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier]@unchecked) = - if (progressInfo.chainSwitchingNeeded) { - branchingPointOpt.map { branchPoint => - if (!state.version.sameElements(branchPoint)) { - val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]){ - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - context.system.actorSelection("/user/miner") ! DisableMining - state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) - } else Success(state) -> IndexedSeq() - }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) - } else Success(state) -> suffixApplied - stateToApplyTry match { - case Success(stateToApply) => - context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) - val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) - val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u, modToApply) => - val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < encrySettings.constants.MaxRollbackDepth * 2 - if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { - case Right(stateAfterApply) => - influxRef.foreach(ref => modToApply match { - case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) - case _ => - }) - val newHis: History = history.reportModifierIsValid(modToApply) - dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) - modToApply match { - case header: Header => - val requiredHeight: Int = header.height - encrySettings.constants.MaxRollbackDepth - if (requiredHeight % encrySettings.constants.SnapshotCreationHeight == 0) { - newHis.lastAvailableManifestHeight = requiredHeight - logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") - } - case _ => - } - newHis.getHeaderOfBestBlock.foreach { header: Header => - val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) - val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) - val isCorrectCreationHeight: Boolean = - header.height % encrySettings.constants.SnapshotCreationHeight == 0 - val isGenesisHeader: Boolean = header.height == encrySettings.constants.GenesisHeight - if (encrySettings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && - !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { - val startTime = System.currentTimeMillis() - logger.info(s"Start chunks creation for new snapshot") - import encry.view.state.avlTree.utils.implicits.Instances._ - val chunks: List[SnapshotChunk] = - AvlTree.getChunks( - stateAfterApply.tree.rootNode, - currentChunkHeight = encrySettings.snapshotSettings.chunkDepth, - stateAfterApply.tree.avlStorage - ) - system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) - potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds - logger.info(s"State tree successfully processed for snapshot. " + - s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - } - } - if (encrySettings.node.mining && progressInfo.chainSwitchingNeeded) context.system.actorSelection("/user/miner") ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) - if (newHis.getBestHeaderId.exists(bestHeaderId => - newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) - )) newHis.isFullChainSynced = true - influxRef.foreach { ref => - logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") - ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) - val isBlock: Boolean = modToApply match { - case _: Block => true - case _: Payload => true - case _ => false - } - if (isBlock) ref ! ModifierAppendedToState(success = true) - } - UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) - case Left(e) => - logger.info(s"Application to state failed cause $e") - val (newHis: History, newProgressInfo: ProgressInfo) = - history.reportModifierIsInvalid(modToApply) - context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) - UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) - } else u - } - uf.failedMod match { - case Some(_) => - uf.history.updateIdsForSyncInfo() - updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) - case None => (uf.history, uf.state, uf.suffix) - } - case Failure(e) => - context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) - EncryApp.forceStopApplication(500, s"Rollback failed: $e") - } - } - - def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = - if (!nodeView.history.isModifierDefined(pmod.id)) { - logger.debug(s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history.") - val startAppHistory = System.currentTimeMillis() - if (encrySettings.influxDB.isDefined) context.system - .actorSelection("user/statsSender") ! - StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) - nodeView.history.append(pmod) match { - case Right((historyBeforeStUpdate, progressInfo)) => - logger.info(s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history.") - logger.debug(s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}") - if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref => - ref ! EndOfApplyingModifier(pmod.id) - val isHeader: Boolean = pmod match { - case _: Header => true - case _: Payload => false - } - ref ! ModifierAppendedToHistory(isHeader, success = true) - } - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && - historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") - logger.info(s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}") - historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight) - .foreach { h => - system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( - historyBeforeStUpdate.lastAvailableManifestHeight, - Algos.hash(h.stateRoot ++ h.id) - ) - } - } - logger.info(s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo") - if (progressInfo.toApply.nonEmpty) { - logger.info(s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}") - val startPoint: Long = System.currentTimeMillis() - val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = - updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) - if (newHistory.isHeadersChainSynced) system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced - if (encrySettings.influxDB.isDefined) - context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) - sendUpdatedInfoToMemoryPool(progressInfo.toRemove) - if (progressInfo.chainSwitchingNeeded) - nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get - blocksApplied.foreach(nodeView.wallet.scanPersistent) - logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") - if (encrySettings.influxDB.isDefined) newHistory.getBestHeader.foreach(header => - context.actorSelection("/user/statsSender") ! BestHeaderInChain(header)) - if (newHistory.isFullChainSynced) { - logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") - ModifiersCache.setChainSynced() - system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced - system.actorSelection("/user/miner") ! FullBlockChainIsSynced - } - updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) - } else { - influxRef.foreach { ref => - logger.info(s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}") - ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) - } - if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) - context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) - logger.info(s"\nProgress info is empty") - updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) - } - case Left(e) => - logger.debug(s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + - s" to history caused $e") - context.system.eventStream.publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) - } - } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") - - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { - val rolledBackTxs: IndexedSeq[Transaction] = toRemove - .flatMap(extractTransactions) - .toIndexedSeq - if (rolledBackTxs.nonEmpty) - memoryPoolRef ! RolledBackTransactions(rolledBackTxs) - } - - def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { - case b: Block => b.payload.txs - case p: Payload => p.txs - case _ => Seq.empty[Transaction] - } - - def genesisState(influxRef: Option[ActorRef] = None): NodeView = { - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdir() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, encrySettings, influxRef) - val history: History = History.readOrGenerate(encrySettings, timeProvider) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(encrySettings), EncryWallet.getKeysDir(encrySettings), encrySettings) - NodeView(history, state, wallet) - } - - def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = if (History.getHistoryIndexDir(encrySettings).listFiles.nonEmpty) - try { - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - val history: History = History.readOrGenerate(encrySettings, timeProvider) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(encrySettings), EncryWallet.getKeysDir(encrySettings), encrySettings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, encrySettings, influxRef), history, influxRef - ) - history.updateIdsForSyncInfo() - logger.info(s"History best block height: ${history.getBestBlockHeight}") - logger.info(s"History best header height: ${history.getBestHeaderHeight}") - Some(NodeView(history, state, wallet)) - } catch { - case ex: Throwable => - logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") - new File(encrySettings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - Some(genesisState(influxRef)) - } else { - None - } - - def getRecreatedState(version: Option[VersionTag] = None, - digest: Option[ADDigest] = None, - influxRef: Option[ActorRef]): UtxoState = { - val dir: File = UtxoState.getStateDir(encrySettings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - UtxoState.create(stateDir, rootsDir, encrySettings, influxRef) - } - - def restoreConsistentState(stateIn: UtxoState, history: History, influxRefActor: Option[ActorRef]): UtxoState = - (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { - case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => - logger.info(s"State and history are both empty on startup") - stateIn - case (_, None, _, _) => - logger.info(s"State and history are inconsistent." + - s" History is empty on startup, rollback state to genesis.") - getRecreatedState(influxRef = influxRefActor) - case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => - val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) - val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) - logger.info(s"State and history are inconsistent." + - s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers") - val additionalBlocks = (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]){ - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") - rollbackId.map(_ => state.restore(additionalBlocks).get) - .getOrElse(getRecreatedState(influxRef = influxRefActor)) - } - - override def close(): Unit = { - nodeView.history.close() - nodeView.state.close() - nodeView.wallet.close() - } } object NodeViewHolder { From 35dc809c44f957e7d025506114aa2abd03d78601 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 14:43:52 +0300 Subject: [PATCH 007/119] added full-filled receive function --- src/main/scala/encry/nvg/NodeViewHolder.scala | 100 ++++++++++++------ .../scala/encry/view/NodeViewHolder.scala | 52 +-------- 2 files changed, 68 insertions(+), 84 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f83295b81d..194cb16c6f 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -6,46 +6,27 @@ import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.EncryApp.system -import encry.api.http.DataHolderForApi import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ - ChangedHistory, - ChangedState, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings -import encry.stats.StatsSender.{ - BestHeaderInChain, - EndOfApplyingModifier, - HeightStatistics, - ModifierAppendedToHistory, - ModifierAppendedToState, - StartApplyingModifier, - StateUpdating, - TransactionsInBlock -} +import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.NodeViewHolder.ReceivableMessages.{ + CreateAccountManagerFromSeed, + LocallyGeneratedModifier, + ModifierFromRemote +} import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder.{ - HeaderChainIsSynced, - RequiredManifestHeightAndId, - SnapshotChunk, - TreeChunks -} -import encry.view.history.{ History, HistoryReader } -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.view.fast.sync.SnapshotHolder._ +import encry.view.history.storage.HistoryStorage +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet @@ -58,7 +39,6 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } import scala.collection.{ mutable, IndexedSeq, Seq } -import scala.concurrent.ExecutionContextExecutor import scala.util.{ Failure, Success, Try } class NodeViewHolder( @@ -69,14 +49,68 @@ class NodeViewHolder( with StrictLogging with AutoCloseable { - import context.dispatcher - var nodeView: NodeView = restoreState().getOrElse(genesisState) var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case ModifierFromRemote(modifier) => + case ModifierFromRemote(modifier: PersistentModifier) => + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(key(modifier.id)) + if (isInHistory || isInCache) + logger.info( + s"Modifier ${modifier.encodedId} can't be placed into the cache cause:" + + s" contains in cache: $isInCache, contains in history: $isInHistory" + ) + else ModifiersCache.put(key(modifier.id), modifier, nodeView.history) + computeApplications() + + case LocallyGeneratedModifier(modifier: PersistentModifier) => + val startTime: Long = System.currentTimeMillis() + logger.info(s"Got locally generated modifier ${modifier.encodedId} of type ${modifier.modifierTypeId}") + modifier match { + case block: Block => + pmodModify(block.header, isLocallyGenerated = true) + pmodModify(block.payload, isLocallyGenerated = true) + } + logger.debug( + s"Processing time of modifier ${modifier.encodedId} is ${(System.currentTimeMillis() - startTime) / 1000}s." + ) + + case FastSyncFinished(state, wallet) => + logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") + nodeView.state.tree.avlStorage.close() + nodeView.wallet.close() + FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) + FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) + FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) + logger.info(s"Updated best block in fast sync mod. Updated state height.") + val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { + override val settings: EncryAppSettings = settings + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider + override val historyStorage: HistoryStorage = nodeView.history.historyStorage + } + newHistory.fastSyncInProgress.fastSyncVal = false + newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar( + nodeView.history.blockDownloadProcessor.minimalBlockHeight + ) + newHistory.isHeadersChainSyncedVar = true + updateNodeView( + updatedHistory = Some(newHistory), + updatedState = Some(state), + updatedVault = Some(wallet) + ) + system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone + logger.info(s"Fast sync finished successfully!") + + case CreateAccountManagerFromSeed(seed) => + val newAccount = nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) + updateNodeView(updatedVault = newAccount.toOption) + sender() ! newAccount + + case RemoveRedundantManifestIds => potentialManifestIds = List.empty + } //todo refactor loop diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 404f9a52ae..9f3a060876 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -84,57 +84,7 @@ class NodeViewHolder(memoryPoolRef: ActorRef, var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case CreateAccountManagerFromSeed(seed) => - val newAccount = nodeView.wallet.addAccount(seed, encrySettings.wallet.map(_.password).get, nodeView.state) - updateNodeView(updatedVault = newAccount.toOption) - sender() ! newAccount - case FastSyncFinished(state, wallet) => - logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") - nodeView.state.tree.avlStorage.close() - nodeView.wallet.close() - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/tmpDirState")) - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/keysTmp")) - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/walletTmp")) - logger.info(s"Updated best block in fast sync mod. Updated state height.") - val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = encrySettings - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider - override val historyStorage: HistoryStorage = nodeView.history.historyStorage - } - newHistory.fastSyncInProgress.fastSyncVal = false - newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar(nodeView.history.blockDownloadProcessor.minimalBlockHeight) - newHistory.isHeadersChainSyncedVar = true - updateNodeView( - updatedHistory = Some(newHistory), - updatedState = Some(state), - updatedVault = Some(wallet) - ) - system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone - logger.info(s"Fast sync finished successfully!") - case RemoveRedundantManifestIds => potentialManifestIds = List.empty - case ModifierFromRemote(mod) => - val isInHistory: Boolean = nodeView.history.isModifierDefined(mod.id) - val isInCache: Boolean = ModifiersCache.contains(key(mod.id)) - if (isInHistory || isInCache) - logger.info(s"Received modifier of type: ${mod.modifierTypeId} ${Algos.encode(mod.id)} " + - s"can't be placed into cache cause of: inCache: ${!isInCache}.") - else ModifiersCache.put(key(mod.id), mod, nodeView.history) - computeApplications() - - case lm: LocallyGeneratedModifier => - logger.debug(s"Start processing LocallyGeneratedModifier message on NVH.") - val startTime = System.currentTimeMillis() - logger.info(s"Got locally generated modifier ${lm.pmod.encodedId} of type ${lm.pmod.modifierTypeId}") - lm.pmod match { - case block: Block => - pmodModify(block.header, isLocallyGenerated = true) - pmodModify(block.payload, isLocallyGenerated = true) - case anyMod => - pmodModify(anyMod, isLocallyGenerated = true) - } - logger.debug(s"Time processing of msg LocallyGeneratedModifier with mod of type ${lm.pmod.modifierTypeId}:" + - s" with id: ${Algos.encode(lm.pmod.id)} -> ${System.currentTimeMillis() - startTime}") + case GetDataFromCurrentView(f) => f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { From 6abbb600dc781faa3bfe4d85251d9180e6b79e66 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 14:46:40 +0300 Subject: [PATCH 008/119] add some additions to new peers keeper --- .../encry/network/NetworkController.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 21 +++--- src/main/scala/encry/network/PK.scala | 71 ++++++++++++++++++- .../scala/encry/network/PeersKeeper.scala | 10 +-- .../network/ConnectWithNewPeerTests.scala | 18 ++--- 5 files changed, 94 insertions(+), 28 deletions(-) diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index af2d242cac..92e43451d4 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -93,7 +93,7 @@ class NetworkController(networkSettings: NetworkSettings, logger.info(s"Network controller got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") - peersKeeper ! VerifyConnection(remote, sender()) + peersKeeper ! NewConnection(remote, sender()) case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a1ba21cf58..4602df1311 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -12,11 +12,12 @@ import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} -import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, OutgoingConnectionFailed, PeerForConnection, VerifyConnection} -import encry.settings.NetworkSettings +import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed, PeerForConnection} +import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage -class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging { +class NetworkRouter(settings: NetworkSettings, + blackListSettings: BlackListSettings) extends Actor with StrictLogging { import context.system @@ -24,7 +25,7 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - val peersKeeper = context.system.actorOf(PK.props(), "peersKeeper") + val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress @@ -70,7 +71,12 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging logger.info(s"Network router got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") - peersKeeper ! VerifyConnection(remote, sender()) + peersKeeper ! NewConnection(remote, sender()) + + case HandshakedDone(remote) => + logger.info(s"Network controller got approvement from peer handler about successful handshake. " + + s"Sending to peerKeeper connected peer.") + peersKeeper ! HandshakedDone(remote) case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") @@ -80,11 +86,6 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging ) peerConnectionHandler ! StartInteraction - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - case ConnectionStopped(peer) => logger.info(s"Network controller got signal about breaking connection with: $peer. " + s"Sending to peerKeeper actual information.") diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 7adda9e5db..304caf540e 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,13 +2,19 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, Props} +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeersKeeper.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import scala.util.Try class PK(networkSettings: NetworkSettings, - blacklistSettings: BlackListSettings) extends Actor { + blacklistSettings: BlackListSettings) extends Actor with StrictLogging { + + val connectWithOnlyKnownPeers: Boolean = networkSettings.connectOnlyWithKnownPeers.getOrElse(true) var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() @@ -17,7 +23,66 @@ class PK(networkSettings: NetworkSettings, var knownPeers: Set[InetAddress] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet - override def receive: Receive = ??? + var outgoingConnections: Set[InetSocketAddress] = Set.empty + + var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty + + var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap + + override def receive: Receive = { + case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => + logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + + s"Current known peers: ${knownPeers.mkString(",")}") + val notConnectedYet: Boolean = !connectedPeers.contains(remote) + val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) + if (notConnectedYet && notBannedPeer) { + logger.info(s"Peer: $remote is available to setup connect with.") + if (outgoingConnections.contains(remote)) { + logger.info(s"Got outgoing connection.") + outgoingConnections -= remote + sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) + } + else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { + logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") + sender() ! ConnectionVerified(remote, remoteConnection, Incoming) + } + else if (connectWithOnlyKnownPeers) + logger.info(s"Got incoming connection but we can connect only with known peers.") + else { + logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") + sender() ! ConnectionVerified(remote, remoteConnection, Incoming) + } + } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + + s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") + + case NewConnection(remote, _) => + logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + + s"bigger than possible or isSelf: ${isSelf(remote)}.") + + case HandshakedDone(connectedPeer) => + logger.info(s"Peers keeper got approvement about finishing a handshake." + + s" Initializing new peer: ${connectedPeer.socketAddress}") + connectedPeers = connectedPeers.initializePeer(connectedPeer) + logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + + s"${awaitingHandshakeConnections.mkString(",")}.") + awaitingHandshakeConnections -= connectedPeer.socketAddress + peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) + logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + + s" Current collection is: ${peersForConnection.keys.mkString(",")}.") + + case ConnectionStopped(peer) => + logger.info(s"Connection stopped for: $peer.") + awaitingHandshakeConnections -= peer + connectedPeers = connectedPeers.removePeer(peer) + if (blackList.contains(peer.getAddress)) { + peersForConnection -= peer + logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + + s"Current is: ${peersForConnection.mkString(",")}.") + } + + } def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 584b0b225e..8bdf174b69 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -108,7 +108,7 @@ class PeersKeeper(settings: EncryAppSettings, case RequestPeerForConnection => logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") - case VerifyConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => + case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + s"Current known peers: ${knownPeers.mkString(",")}") @@ -134,7 +134,7 @@ class PeersKeeper(settings: EncryAppSettings, } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") - case VerifyConnection(remote, remoteConnection) => + case NewConnection(remote, remoteConnection) => logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + s"bigger than possible or isSelf: ${isSelf(remote)}.") @@ -310,8 +310,8 @@ object PeersKeeper { sealed trait PeerCommandHelper - final case class VerifyConnection(peer: InetSocketAddress, - remoteConnection: ActorRef) + final case class NewConnection(peer: InetSocketAddress, + remoteConnection: ActorRef) final case class ConnectionVerified(peer: InetSocketAddress, remoteConnection: ActorRef, @@ -353,7 +353,7 @@ object PeersKeeper { case AccumulatedPeersStatistic(_) => 1 case BanPeer(_, _) => 1 case SendLocalSyncInfo => 1 - case VerifyConnection(_, _) => 2 + case NewConnection(_, _) => 2 case HandshakedDone(_) => 2 case ConnectionStopped(_) => 2 case OutgoingConnectionFailed(_) => 2 diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index c7b4e905c6..bdb2126515 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -219,7 +219,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -239,7 +239,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -260,7 +260,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -275,7 +275,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) peersKeeper.stop() } @@ -290,7 +290,7 @@ class ConnectWithNewPeerTests extends WordSpecLike "test-peer", Some(remoteAddress), System.currentTimeMillis())) peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -305,7 +305,7 @@ class ConnectWithNewPeerTests extends WordSpecLike "test-peer", Some(remoteAddress), System.currentTimeMillis())) peersKeeper ! HandshakedDone(connectedPeer) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -315,7 +315,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -324,7 +324,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteConnectionTestProbe: TestProbe = TestProbe() val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection( + networkController.send(peersKeeper, NewConnection( new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() @@ -336,7 +336,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper ! RequestPeerForConnection peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) networkController.expectMsg( ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) } From aa18607864e585f62d79044a7f26edc689d11a16 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:07:57 +0300 Subject: [PATCH 009/119] nvh some cleanups --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 + .../scala/encry/nvg/ModifiersValidator.scala | 7 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 65 ++++++++++--------- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f1af14f4b0..887dd813cb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -10,6 +10,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.view.fast.sync.SnapshotHolder.FastSyncDone import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -41,6 +42,7 @@ class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) networkMessagesProcessor ! newReader case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 512620a119..48ce5b076e 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -15,9 +15,8 @@ import encry.network.BlackList.BanReason.{ import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } @@ -40,7 +39,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { logger.debug(s"Modifier ${modifier.encodedId} is valid.") - nodeViewHolderRef ! ModifierFromRemote(modifier) + nodeViewHolderRef ! ValidatedModifier(modifier) } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) @@ -100,6 +99,8 @@ object ModifiersValidator { remote: ConnectedPeer ) + final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 194cb16c6f..378d770626 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,15 +1,13 @@ package encry.nvg import java.io.File - import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.EncryApp.system import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings import encry.stats.StatsSender._ @@ -17,11 +15,7 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ - CreateAccountManagerFromSeed, - LocallyGeneratedModifier, - ModifierFromRemote -} +import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ @@ -37,13 +31,12 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } - import scala.collection.{ mutable, IndexedSeq, Seq } import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, - timeProvider: NetworkTimeProvider, + ntp: NetworkTimeProvider, influxRef: Option[ActorRef] ) extends Actor with StrictLogging @@ -54,41 +47,48 @@ class NodeViewHolder( var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case ModifierFromRemote(modifier: PersistentModifier) => - val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) - val isInCache: Boolean = ModifiersCache.contains(key(modifier.id)) + case ValidatedModifier(modifier: PersistentModifier) => + val startTime: Long = System.currentTimeMillis() + val wrappedKey: mutable.WrappedArray.ofByte = key(modifier.id) + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) logger.info( - s"Modifier ${modifier.encodedId} can't be placed into the cache cause:" + - s" contains in cache: $isInCache, contains in history: $isInHistory" + s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + + s"contains in cache: $isInCache, contains in history: $isInHistory." ) - else ModifiersCache.put(key(modifier.id), modifier, nodeView.history) + else ModifiersCache.put(wrappedKey, modifier, nodeView.history) computeApplications() + logger.debug( + s"Time of processing validated modifier with id: ${modifier.encodedId} " + + s"is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) case LocallyGeneratedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - logger.info(s"Got locally generated modifier ${modifier.encodedId} of type ${modifier.modifierTypeId}") + logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => pmodModify(block.header, isLocallyGenerated = true) pmodModify(block.payload, isLocallyGenerated = true) } logger.debug( - s"Processing time of modifier ${modifier.encodedId} is ${(System.currentTimeMillis() - startTime) / 1000}s." + s"Time of process locally generated modifier with id: ${modifier.encodedId} " + + s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) case FastSyncFinished(state, wallet) => - logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") + val startTime: Long = System.currentTimeMillis() + logger.info(s"Node view holder got a signal about finishing fast sync process.") nodeView.state.tree.avlStorage.close() nodeView.wallet.close() FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) - logger.info(s"Updated best block in fast sync mod. Updated state height.") val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { override val settings: EncryAppSettings = settings override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider + override val timeProvider: NetworkTimeProvider = ntp override val historyStorage: HistoryStorage = nodeView.history.historyStorage } newHistory.fastSyncInProgress.fastSyncVal = false @@ -101,24 +101,29 @@ class NodeViewHolder( updatedState = Some(state), updatedVault = Some(wallet) ) - system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone - logger.info(s"Fast sync finished successfully!") + context.parent ! FastSyncDone + logger.debug( + s"Time of processing fast sync done message is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) case CreateAccountManagerFromSeed(seed) => - val newAccount = nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) + val newAccount: Either[String, EncryWallet] = + nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) updateNodeView(updatedVault = newAccount.toOption) sender() ! newAccount - case RemoveRedundantManifestIds => potentialManifestIds = List.empty + case RemoveRedundantManifestIds => + potentialManifestIds = List.empty } //todo refactor loop def computeApplications(): Unit = { - val mods = ModifiersCache.popCandidate(nodeView.history) - if (mods.nonEmpty) { + val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) + if (modifiers.nonEmpty) { + logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - mods.foreach(mod => pmodModify(mod)) + modifiers.foreach(mod => pmodModify(mod)) computeApplications() } else Unit } @@ -395,7 +400,7 @@ class NodeViewHolder( rootsDir.mkdir() assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - val history: History = History.readOrGenerate(settings, timeProvider) + val history: History = History.readOrGenerate(settings, ntp) val wallet: EncryWallet = EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) NodeView(history, state, wallet) @@ -408,7 +413,7 @@ class NodeViewHolder( stateDir.mkdirs() val rootsDir: File = UtxoState.getRootsDir(settings) rootsDir.mkdir() - val history: History = History.readOrGenerate(settings, timeProvider) + val history: History = History.readOrGenerate(settings, ntp) val wallet: EncryWallet = EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) val state: UtxoState = restoreConsistentState( From 25622bb78bb92b80afcf4b814e94b60db69d0482 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:48:54 +0300 Subject: [PATCH 010/119] nvh improved --- .../scala/encry/nvg/IntermediaryNVH.scala | 23 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 191 ++++++++---------- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 887dd813cb..8701259d2e 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -3,14 +3,22 @@ package encry.nvg import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings -import encry.view.fast.sync.SnapshotHolder.FastSyncDone +import encry.view.NodeViewHolder.DownloadRequest +import encry.view.fast.sync.SnapshotHolder.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -40,9 +48,16 @@ class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 378d770626..8b9d90f0a2 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,10 +1,13 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } + +import cats.syntax.option._ +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier @@ -15,24 +18,25 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } +import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.view.NodeViewHolder.{DownloadRequest, UpdateInformation} import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } -import scala.collection.{ mutable, IndexedSeq, Seq } -import scala.util.{ Failure, Success, Try } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} + +import scala.collection.{IndexedSeq, Seq, mutable} +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -69,8 +73,8 @@ class NodeViewHolder( logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => - pmodModify(block.header, isLocallyGenerated = true) - pmodModify(block.payload, isLocallyGenerated = true) + applyModfier(block.header, isLocallyGenerated = true) + applyModfier(block.payload, isLocallyGenerated = true) } logger.debug( s"Time of process locally generated modifier with id: ${modifier.encodedId} " + @@ -97,9 +101,9 @@ class NodeViewHolder( ) newHistory.isHeadersChainSyncedVar = true updateNodeView( - updatedHistory = Some(newHistory), - updatedState = Some(state), - updatedVault = Some(wallet) + updatedHistory = newHistory.some, + updatedState = state.some, + updatedVault = wallet.some ) context.parent ! FastSyncDone logger.debug( @@ -122,8 +126,7 @@ class NodeViewHolder( val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) if (modifiers.nonEmpty) { logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") - logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - modifiers.foreach(mod => pmodModify(mod)) + modifiers.foreach(applyModfier(_)) computeApplications() } else Unit } @@ -146,35 +149,35 @@ class NodeViewHolder( def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = pi.toDownload.foreach { - case (tid, id) => - if (tid != Transaction.modifierTypeId) - logger.info( - s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + - s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}." - ) + case (tid: ModifierTypeId, id: ModifierId) => + logger.info( + s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid." + + s" Previous modifier is ${previousModifier.map(Algos.encode)}." + ) if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) - system.actorSelection("/user/nodeViewSynchronizer") ! DownloadRequest(tid, id, previousModifier) - else - logger.info( - s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false" - ) + context.parent ! DownloadRequest(tid, id, previousModifier) + else logger.info(s"Ignore sending request for payload ${Algos.encode(id)} because full chain is not synced.") } - def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], - rollbackPoint: ModifierId): IndexedSeq[PersistentModifier] = { + def trimChainSuffix( + suffix: IndexedSeq[PersistentModifier], + rollbackPoint: ModifierId + ): IndexedSeq[PersistentModifier] = { val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) - if (idx == -1) IndexedSeq() else suffix.drop(idx) + if (idx == -1) IndexedSeq.empty else suffix.drop(idx) } @scala.annotation.tailrec - private def updateState(history: History, - state: UtxoState, - progressInfo: ProgressInfo, - suffixApplied: IndexedSeq[PersistentModifier], - isLocallyGenerated: Boolean = false): (History, UtxoState, Seq[PersistentModifier]) = { - logger.info(s"\nStarting updating state in updateState function!") + private def updateState( + history: History, + state: UtxoState, + progressInfo: ProgressInfo, + suffixApplied: IndexedSeq[PersistentModifier], + isLocallyGenerated: Boolean = false + ): (History, UtxoState, Seq[PersistentModifier]) = { + logger.info(s"Starting updating state in updateState function!") if (!isLocallyGenerated) progressInfo.toApply.foreach { - case header: Header => requestDownloads(progressInfo, Some(header.id)) + case header: Header => requestDownloads(progressInfo, header.id.some) case _ => requestDownloads(progressInfo, None) } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) @@ -189,10 +192,10 @@ class NodeViewHolder( val blockAtHeight = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.system.actorSelection("/user/miner") ! DisableMining + context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) - } else Success(state) -> IndexedSeq() + } else Success(state) -> IndexedSeq.empty }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) } else Success(state) -> suffixApplied stateToApplyTry match { @@ -201,7 +204,8 @@ class NodeViewHolder( val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u, modToApply) => - val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 + val saveRootNodesFlag: Boolean = + (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { case Right(stateAfterApply) => influxRef.foreach( @@ -239,7 +243,7 @@ class NodeViewHolder( currentChunkHeight = settings.snapshotSettings.chunkDepth, stateAfterApply.tree.avlStorage ) - system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) + context.parent ! TreeChunks(chunks, potentialManifestId) potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds logger.info( s"State tree successfully processed for snapshot. " + @@ -248,7 +252,7 @@ class NodeViewHolder( } } if (settings.node.mining && progressInfo.chainSwitchingNeeded) - context.system.actorSelection("/user/miner") ! StartMining + context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) if (newHis.getBestHeaderId.exists( bestHeaderId => @@ -270,7 +274,7 @@ class NodeViewHolder( val (newHis: History, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(modToApply) context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) - UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) + UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) } else u } uf.failedMod match { @@ -285,98 +289,81 @@ class NodeViewHolder( } } - def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = - if (!nodeView.history.isModifierDefined(pmod.id)) { + def applyModfier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + if (!nodeView.history.isModifierDefined(modifier.id)) { logger.debug( - s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." ) - val startAppHistory = System.currentTimeMillis() - if (settings.influxDB.isDefined) - context.system - .actorSelection("user/statsSender") ! - StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) - nodeView.history.append(pmod) match { + val startApplicationToTheHistory: Long = System.currentTimeMillis() + influxRef.foreach(_ ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis())) + nodeView.history.append(modifier) match { case Right((historyBeforeStUpdate, progressInfo)) => logger.info( - s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." - ) - logger.debug( - s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}" + s"Successfully applied modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the history. " + + s"Time of applying is: ${(System.currentTimeMillis() - startApplicationToTheHistory) / 1000}s." ) - if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref => - ref ! EndOfApplyingModifier(pmod.id) - val isHeader: Boolean = pmod match { + if (modifier.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() + influxRef.foreach { ref: ActorRef => + ref ! EndOfApplyingModifier(modifier.id) + ref ! ModifierAppendedToHistory(modifier match { case _: Header => true case _: Payload => false - } - ref ! ModifierAppendedToHistory(isHeader, success = true) + }, success = true) } - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && + if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && + modifier.modifierTypeId == Payload.modifierTypeId && historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") logger.info( - s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" + s"getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}. " + + s"heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" ) historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { - h => - system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( + h: Header => + context.parent ! RequiredManifestHeightAndId( historyBeforeStUpdate.lastAvailableManifestHeight, Algos.hash(h.stateRoot ++ h.id) ) } } logger.info( - s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo" + s"Going to apply modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the state. " + + s"Progress info is: $progressInfo." ) if (progressInfo.toApply.nonEmpty) { - logger.info( - s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}" - ) val startPoint: Long = System.currentTimeMillis() + logger.info(s"Progress info is non empty. To apply is: ${progressInfo.toApply.map(_.encodedId)}.") val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) - if (newHistory.isHeadersChainSynced) - system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced - if (settings.influxDB.isDefined) - context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) + if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced + influxRef.foreach(_ ! StateUpdating(System.currentTimeMillis() - startPoint)) sendUpdatedInfoToMemoryPool(progressInfo.toRemove) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) - logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") - if (settings.influxDB.isDefined) - newHistory.getBestHeader.foreach( - header => context.actorSelection("/user/statsSender") ! BestHeaderInChain(header) - ) + logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") + for { + ref <- influxRef + header <- newHistory.getBestHeader + } yield ref ! BestHeaderInChain(header) if (newHistory.isFullChainSynced) { - logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") + logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() - system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced - system.actorSelection("/user/miner") ! FullBlockChainIsSynced + context.parent ! FullBlockChainIsSynced } - updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) + updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { - influxRef.foreach { ref => - logger.info( - s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}" - ) - ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) - } - if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) - context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) - logger.info(s"\nProgress info is empty") - updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) + logger.info(s"Progress info is empty.") + influxRef.foreach(_ ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height)) + if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e) => - logger.debug( - s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + - s" to history caused $e" - ) + logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") context.system.eventStream - .publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) + .publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage)))) } - } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") + } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) @@ -424,12 +411,12 @@ class NodeViewHolder( history.updateIdsForSyncInfo() logger.info(s"History best block height: ${history.getBestBlockHeight}") logger.info(s"History best header height: ${history.getBestHeaderHeight}") - Some(NodeView(history, state, wallet)) + NodeView(history, state, wallet).some } catch { case ex: Throwable => logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - Some(genesisState) + genesisState.some } else { None } From f52462d5ce99cc99c14e16113f23f27967b73b9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:50:20 +0300 Subject: [PATCH 011/119] added apply to nvh --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8701259d2e..8a94b2d588 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -12,6 +12,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest import encry.view.fast.sync.SnapshotHolder.{ FastSyncDone, @@ -22,12 +23,18 @@ import encry.view.fast.sync.SnapshotHolder.{ import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId -class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) extends Actor with StrictLogging { +class IntermediaryNVH( + settings: EncryAppSettings, + intermediaryNetwork: ActorRef, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging { val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") val nodeViewHolder: ActorRef = - context.actorOf(NodeViewHolder.props, name = "Node-view-holder") + context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = context.actorOf( BalancingPool(5) From ddb10b41d452c4979292ac62fdb655d68225d09c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:55:28 +0300 Subject: [PATCH 012/119] more additions --- .../scala/encry/nvg/IntermediaryNVH.scala | 17 +++++++------ src/main/scala/encry/nvg/NodeViewHolder.scala | 24 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8a94b2d588..9bdd31a8e6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,12 +1,13 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef } +import akka.actor.{Actor, ActorRef} import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.BlackList.BanReason.CorruptedSerializedBytes import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -14,13 +15,9 @@ import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest -import encry.view.fast.sync.SnapshotHolder.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} +import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} import encry.view.history.HistoryReader +import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( @@ -65,6 +62,8 @@ class IntermediaryNVH( case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 8b9d90f0a2..2c825fec9f 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -3,11 +3,12 @@ package encry.nvg import java.io.File import cats.syntax.option._ -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier @@ -18,25 +19,26 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.view.NodeViewHolder.{DownloadRequest, UpdateInformation} +import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} -import scala.util.{Failure, Success, Try} +import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -216,7 +218,7 @@ class NodeViewHolder( } ) val newHis: History = history.reportModifierIsValid(modToApply) - //dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) + context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) modToApply match { case header: Header => val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth @@ -371,7 +373,7 @@ class NodeViewHolder( val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq - //if (rolledBackTxs.nonEmpty) memoryPoolRef ! RolledBackTransactions(rolledBackTxs) + if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) } def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { From 5e446d4e26991bfffd83188321413193cfbf9aa4 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:10:57 +0300 Subject: [PATCH 013/119] add new pipelines to message builder --- .../scala/encry/network/MessageBuilder.scala | 25 ++++++++++++++++--- src/main/scala/encry/network/Messages.scala | 9 ++++--- .../scala/encry/network/NetworkRouter.scala | 19 ++++---------- src/main/scala/encry/network/PK.scala | 11 +++++--- .../scala/encry/network/PeersKeeper.scala | 19 ++++++++------ 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 5efffac15a..c1156ba322 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -1,13 +1,18 @@ package encry.network +import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout -import encry.network.MessageBuilder.GetPeers +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.PeerConnectionHandler.ConnectedPeer +import org.encryfoundation.common.network.BasicMessagesRepo.{ModifiersNetworkMessage, RequestModifiersNetworkMessage} import scala.concurrent.duration._ +import scala.util.Try case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { @@ -17,9 +22,20 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends override def receive: Receive = { case RequestFromLocal(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + } + } case SendSyncInfo(syncInfo) => - (peersKeeper ? GetPeers).mapTo[List[ActorRef]].map { peers => - peers.foreach(_ ! syncInfo) + (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => + peers.foreach(_.handlerRef ! syncInfo) + } + case ResponseFromLocal(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) + } } } } @@ -27,6 +43,7 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends object MessageBuilder { case object GetPeers + case class GetPeerInfo(peerIp: InetSocketAddress) def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 193e887081..0a57f29f32 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -1,6 +1,6 @@ package encry.network -import encry.network.PeerConnectionHandler.ConnectedPeer +import java.net.InetSocketAddress import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -8,9 +8,12 @@ object Messages { sealed trait MessageToNetwork object MessageToNetwork { - final case class RequestFromLocal(source: ConnectedPeer, + final case class RequestFromLocal(source: InetSocketAddress, modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId]) extends MessageToNetwork + modifierIds: List[ModifierId]) extends MessageToNetwork final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork + final case class ResponseFromLocal(source: InetSocketAddress, + modifierTypeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 4602df1311..41f7629a7b 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -12,7 +12,8 @@ import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} -import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed, PeerForConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} +import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage @@ -53,7 +54,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg), "peersKeeper") + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } def peersLogic: Receive = { @@ -66,18 +67,11 @@ class NetworkRouter(settings: NetworkSettings, Some(settings.connectionTimeout), pullMode = true ) - case Connected(remote, localAddress) => - logger.info(s"Network router got 'Connected' message from: $remote. " + + logger.info(s"Network controller got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") peersKeeper ! NewConnection(remote, sender()) - - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") val peerConnectionHandler: ActorRef = context.actorOf( @@ -86,10 +80,7 @@ class NetworkRouter(settings: NetworkSettings, ) peerConnectionHandler ! StartInteraction - case ConnectionStopped(peer) => - logger.info(s"Network controller got signal about breaking connection with: $peer. " + - s"Sending to peerKeeper actual information.") - peersKeeper ! ConnectionStopped(peer) + case msg: ConnectionStatusMessages => peersKeeper ! msg case CommandFailed(connect: Connect) => logger.info(s"Failed to connect to: ${connect.remoteAddress}.") diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 304caf540e..d0d13ef2cb 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,11 +2,11 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging -import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.{Incoming, Outgoing} -import encry.network.PeersKeeper.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import scala.util.Try @@ -81,7 +81,10 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } - + case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) + case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { + case (_, info) => sender() ! info.connectedPeer + } } def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 8bdf174b69..7a421d15c7 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -310,18 +310,21 @@ object PeersKeeper { sealed trait PeerCommandHelper - final case class NewConnection(peer: InetSocketAddress, - remoteConnection: ActorRef) + sealed trait ConnectionStatusMessages + object ConnectionStatusMessages { + final case class NewConnection(peer: InetSocketAddress, + remoteConnection: ActorRef) extends ConnectionStatusMessages - final case class ConnectionVerified(peer: InetSocketAddress, - remoteConnection: ActorRef, - ct: ConnectionType) + final case class ConnectionVerified(peer: InetSocketAddress, + remoteConnection: ActorRef, + ct: ConnectionType) extends ConnectionStatusMessages - final case class OutgoingConnectionFailed(peer: InetSocketAddress) + final case class OutgoingConnectionFailed(peer: InetSocketAddress) extends ConnectionStatusMessages - final case class HandshakedDone(peer: ConnectedPeer) + final case class HandshakedDone(peer: ConnectedPeer) extends ConnectionStatusMessages - final case class ConnectionStopped(peer: InetSocketAddress) + final case class ConnectionStopped(peer: InetSocketAddress) extends ConnectionStatusMessages + } case object RequestPeerForConnection From 5deac158ccf84bfd2bb50cfc9998af4206fc83e3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:17:37 +0300 Subject: [PATCH 014/119] nvh more improvements --- .../scala/encry/nvg/IntermediaryNVH.scala | 16 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 44 ++++++- .../scala/encry/view/NodeViewHolder.scala | 122 ++++-------------- 3 files changed, 76 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 9bdd31a8e6..c498667274 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,13 +1,12 @@ package encry.nvg -import akka.actor.{Actor, ActorRef} +import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} -import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -15,7 +14,12 @@ import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest -import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.view.fast.sync.SnapshotHolder.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -63,7 +67,7 @@ class IntermediaryNVH( case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg @ RolledBackTransactions(_) => //+ to memory pool } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 2c825fec9f..baadd2da87 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,8 +2,8 @@ package encry.nvg import java.io.File -import cats.syntax.option._ import akka.actor.{ Actor, ActorRef, Props } +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo @@ -12,15 +12,14 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.NodeView +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage @@ -38,6 +37,7 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } class NodeViewHolder( @@ -48,10 +48,27 @@ class NodeViewHolder( with StrictLogging with AutoCloseable { + import context.dispatcher + var nodeView: NodeView = restoreState().getOrElse(genesisState) var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) + + context.system.scheduler.schedule(1.seconds, 10.seconds) { + logger.info( + s"History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + + s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + + s"State height is: ${nodeView.state.height}.\n " + + s"Cache size is: ${ModifiersCache.size}.\n " + + s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." + ) + } + override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() @@ -487,6 +504,25 @@ object NodeViewHolder { final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) + object ReceivableMessages { + final case class CreateAccountManagerFromSeed(seed: String) extends AnyVal + final case class LocallyGeneratedModifier(pmod: PersistentModifier) extends AnyVal + } + + final case class DownloadRequest( + modifierTypeId: ModifierTypeId, + modifierId: ModifierId, + previousModifier: Option[ModifierId] = None + ) extends NodeViewHolderEvent + + final case class UpdateInformation( + history: History, + state: UtxoState, + failedMod: Option[PersistentModifier], + alternativeProgressInfo: Option[ProgressInfo], + suffix: IndexedSeq[PersistentModifier] + ) + def props( settings: EncryAppSettings, timeProvider: NetworkTimeProvider, diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9f3a060876..514fb93335 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,95 +1,34 @@ package encry.view -import java.io.File - -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import akka.pattern._ +import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.EncryApp.{system, timeProvider} -import encry.api.http.DataHolderForApi -import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} -import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings -import encry.stats.StatsSender._ -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.NetworkTimeProvider -import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages._ import encry.view.NodeViewHolder._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder._ -import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor} -import encry.view.mempool.MemoryPool.RolledBackTransactions -import encry.view.state.UtxoState -import encry.view.state.avlTree.AvlTree -import encry.view.wallet.EncryWallet -import io.iohk.iodb.ByteArrayWrapper -import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import scala.collection.{IndexedSeq, Seq, mutable} -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContextExecutor, Future} -import scala.util.{Failure, Success, Try} +import scala.collection.Seq +import scala.concurrent.Future class NodeViewHolder(memoryPoolRef: ActorRef, influxRef: Option[ActorRef], dataHolder: ActorRef, - encrySettings: EncryAppSettings) extends Actor with StrictLogging with AutoCloseable { - - implicit val exCon: ExecutionContextExecutor = context.dispatcher - - context.system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(nodeView.history) - - dataHolder ! UpdatedHistory(nodeView.history) - dataHolder ! ChangedState(nodeView.state) - dataHolder ! DataHolderForApi.BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) - - influxRef.foreach(ref => context.system.scheduler.schedule(5.second, 5.second) { - logger.info(s"send info. about ${nodeView.history.getBestHeaderHeight} | ${nodeView.history.getBestBlockHeight} | " + - s"${nodeView.state.height} -> best header id ${nodeView.history.getBestHeader.map(_.encodedId)} ->" + - s" best block id ${nodeView.history.getBestBlock.map(_.encodedId)}" + - s" Best header at best block height ${nodeView.history.getBestBlock.flatMap(b => - nodeView.history.getBestHeaderAtHeight(b.header.height) - ).map(l => l.encodedId -> Algos.encode(l.payloadId))}") - }) - - override def preStart(): Unit = logger.info(s"Node view holder started.") - - override def preRestart(reason: Throwable, message: Option[Any]): Unit = { - reason.printStackTrace() - System.exit(100) - } - - context.system.scheduler.schedule(1.seconds, 10.seconds)(logger.info(s"Modifiers cache from NVH: " + - s"${ModifiersCache.size}. Elems: ${ModifiersCache.cache.keys.map(key => Algos.encode(key.toArray)).mkString(",")}")) - - override def postStop(): Unit = { - logger.warn(s"Stopping NodeViewHolder...") - nodeView.history.closeStorage() - } - - var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + encrySettings: EncryAppSettings) + extends Actor + with StrictLogging + with AutoCloseable { override def receive: Receive = { - - case GetDataFromCurrentView(f) => f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender()) - case result => sender() ! result + case result => sender() ! result } case GetNodeViewChanges(history, state, _) => @@ -100,38 +39,34 @@ class NodeViewHolder(memoryPoolRef: ActorRef, logger.info(s"Start processing CompareViews message on NVH.") val startTime = System.currentTimeMillis() val ids: Seq[ModifierId] = modifierTypeId match { - case _ => modifierIds - .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) + case _ => + modifierIds + .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) } - if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got compare view message on NVH from ${peer.socketAddress}." + - s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + - s" Sending RequestFromLocal with ids to $sender." + - s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}.") + if (modifierTypeId != Transaction.modifierTypeId) + logger.debug( + s"Got compare view message on NVH from ${peer.socketAddress}." + + s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + + s" Sending RequestFromLocal with ids to $sender." + + s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}." + ) if (ids.nonEmpty && (modifierTypeId == Header.modifierTypeId || (nodeView.history.isHeadersChainSynced && modifierTypeId == Payload.modifierTypeId))) sender() ! RequestFromLocal(peer, modifierTypeId, ids) - logger.debug(s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}") + logger.debug( + s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}" + ) case SemanticallySuccessfulModifier(_) => - case msg => logger.error(s"Got strange message on nvh: $msg") + case msg => logger.error(s"Got strange message on nvh: $msg") } } object NodeViewHolder { - final case class DownloadRequest(modifierTypeId: ModifierTypeId, - modifierId: ModifierId, - previousModifier: Option[ModifierId] = None) extends NodeViewHolderEvent - case class CurrentView[HIS, MS, VL](history: HIS, state: MS, vault: VL) - case class UpdateInformation(history: History, - state: UtxoState, - failedMod: Option[PersistentModifier], - alternativeProgressInfo: Option[ProgressInfo], - suffix: IndexedSeq[PersistentModifier]) object ReceivableMessages { - case class CreateAccountManagerFromSeed(seed: String) case class GetNodeViewChanges(history: Boolean, state: Boolean, vault: Boolean) @@ -141,15 +76,10 @@ object NodeViewHolder { case class CompareViews(source: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierIds: Seq[ModifierId]) - final case class ModifierFromRemote(serializedModifiers: PersistentModifier) - - case class LocallyGeneratedModifier(pmod: PersistentModifier) - } class NodeViewHolderPriorityQueue(settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator { + extends UnboundedStablePriorityMailbox(PriorityGenerator { case CompareViews(_, _, _) => 0 case PoisonPill => 2 @@ -162,4 +92,4 @@ object NodeViewHolder { dataHolder: ActorRef, settings: EncryAppSettings): Props = Props(new NodeViewHolder(memoryPoolRef, influxRef, dataHolder, settings)) -} \ No newline at end of file +} From a220f71878f36e27b880a2c4f2a60fe627d95df1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:17:58 +0300 Subject: [PATCH 015/119] nvh more improvements --- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index baadd2da87..cb679fed5d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -35,7 +35,6 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } - import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } From b394be40b06083d7d6f015c3d110e7a4eaa3f316 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:23:09 +0300 Subject: [PATCH 016/119] add new pipelines to message builder --- src/main/scala/encry/network/MessageBuilder.scala | 8 ++++++-- src/main/scala/encry/network/Messages.scala | 2 ++ src/main/scala/encry/network/NetworkRouter.scala | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index c1156ba322..4df0ef55a9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -7,9 +7,9 @@ import akka.pattern._ import akka.util.Timeout import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer -import org.encryfoundation.common.network.BasicMessagesRepo.{ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} import scala.concurrent.duration._ import scala.util.Try @@ -37,6 +37,10 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) } } + case BroadcastModifier(modTypeId, modInfo) => + (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => + peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) + } } } diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 0a57f29f32..faf0078240 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -15,5 +15,7 @@ object Messages { final case class ResponseFromLocal(source: InetSocketAddress, modifierTypeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork + final case class BroadcastModifier(modifierTypeId: ModifierTypeId, + modifierId: ModifierId) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 41f7629a7b..7268d69107 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -16,6 +16,7 @@ import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, N import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { @@ -101,5 +102,7 @@ class NetworkRouter(settings: NetworkSettings, object NetworkRouter { + case class ModifierFromNetwork(modTypeId: ModifierTypeId, modId: ModifierId, modBytes: Array[Byte]) + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From 4647145a79f0f23bed2658a3a3b44e85e887da5f Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:26:00 +0300 Subject: [PATCH 017/119] change ModifierFromNetwork --- src/main/scala/encry/network/NetworkRouter.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 7268d69107..df413f773b 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -102,7 +102,10 @@ class NetworkRouter(settings: NetworkSettings, object NetworkRouter { - case class ModifierFromNetwork(modTypeId: ModifierTypeId, modId: ModifierId, modBytes: Array[Byte]) + case class ModifierFromNetwork(source: InetSocketAddress, + modTypeId: ModifierTypeId, + modId: ModifierId, + modBytes: Array[Byte]) def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From 867402bad9d29c7d1b4a2f1e2d9a22356b007ca3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:56:45 +0300 Subject: [PATCH 018/119] more nvh improvements --- .../scala/encry/nvg/IntermediaryNVH.scala | 39 +++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 159 +++++++++--------- src/main/scala/encry/stats/StatsSender.scala | 17 +- 3 files changed, 109 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c498667274..3e3517b746 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,12 +8,18 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier +} import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings +import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.DownloadRequest import encry.view.fast.sync.SnapshotHolder.{ FastSyncDone, HeaderChainIsSynced, @@ -56,18 +62,23 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => //+ to stats sender + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => + case msg @ SemanticallySuccessfulModifier(_) => + case msg @ SemanticallyFailedModification(_, _) => } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index cb679fed5d..da205eb964 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -19,6 +19,7 @@ import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache +import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ @@ -35,6 +36,7 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } + import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } @@ -71,7 +73,7 @@ class NodeViewHolder( override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - val wrappedKey: mutable.WrappedArray.ofByte = key(modifier.id) + val wrappedKey: mutable.WrappedArray.ofByte = toKey(modifier.id) val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) @@ -91,17 +93,17 @@ class NodeViewHolder( logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => - applyModfier(block.header, isLocallyGenerated = true) - applyModfier(block.payload, isLocallyGenerated = true) + applyModifier(block.header, isLocallyGenerated = true) + applyModifier(block.payload, isLocallyGenerated = true) } logger.debug( - s"Time of process locally generated modifier with id: ${modifier.encodedId} " + + s"Time of processing locally generated modifier with id: ${modifier.encodedId} " + s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) - case FastSyncFinished(state, wallet) => + case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => val startTime: Long = System.currentTimeMillis() - logger.info(s"Node view holder got a signal about finishing fast sync process.") + logger.info(s"Got a signal about finishing fast sync process.") nodeView.state.tree.avlStorage.close() nodeView.wallet.close() FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) @@ -124,9 +126,9 @@ class NodeViewHolder( updatedVault = wallet.some ) context.parent ! FastSyncDone - logger.debug( - s"Time of processing fast sync done message is: ${(System.currentTimeMillis() - startTime) / 1000}s." - ) + logger.debug(s"Time of processing FastSyncDone message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") + + case RemoveRedundantManifestIds => potentialManifestIds = List.empty case CreateAccountManagerFromSeed(seed) => val newAccount: Either[String, EncryWallet] = @@ -134,26 +136,23 @@ class NodeViewHolder( updateNodeView(updatedVault = newAccount.toOption) sender() ! newAccount - case RemoveRedundantManifestIds => - potentialManifestIds = List.empty - } //todo refactor loop def computeApplications(): Unit = { val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) if (modifiers.nonEmpty) { - logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") - modifiers.foreach(applyModfier(_)) + logger.info(s"Got new modifiers in compute application function: ${modifiers.map(_.encodedId)}.") + modifiers.foreach(applyModifier(_)) computeApplications() - } else Unit + } else () } - //todo replace with reader + //todo replace outgoing message with history reader def updateNodeView( - updatedHistory: Option[History] = None, - updatedState: Option[UtxoState] = None, - updatedVault: Option[EncryWallet] = None + updatedHistory: Option[History] = none, + updatedState: Option[UtxoState] = none, + updatedVault: Option[EncryWallet] = none ): Unit = { val newNodeView: NodeView = NodeView( updatedHistory.getOrElse(nodeView.history), @@ -165,16 +164,19 @@ class NodeViewHolder( nodeView = newNodeView } - def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = + def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => logger.info( - s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid." + - s" Previous modifier is ${previousModifier.map(Algos.encode)}." + s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + + s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) - if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) + if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) context.parent ! DownloadRequest(tid, id, previousModifier) - else logger.info(s"Ignore sending request for payload ${Algos.encode(id)} because full chain is not synced.") + else + logger.info( + s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." + ) } def trimChainSuffix( @@ -196,20 +198,22 @@ class NodeViewHolder( logger.info(s"Starting updating state in updateState function!") if (!isLocallyGenerated) progressInfo.toApply.foreach { case header: Header => requestDownloads(progressInfo, header.id.some) - case _ => requestDownloads(progressInfo, None) + case _ => requestDownloads(progressInfo, none) } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = if (progressInfo.chainSwitchingNeeded) { - branchingPointOpt.map { branchPoint => + branchingPointOpt.map { branchPoint: VersionTag => if (!state.version.sameElements(branchPoint)) { - val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } + val branchPointHeight: Int = history.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks: List[Block] = + (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { + case (blocks: List[Block], height: Int) => + //todo get best header id instead of best header + val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get + val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) @@ -217,22 +221,19 @@ class NodeViewHolder( }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) } else Success(state) -> suffixApplied stateToApplyTry match { - case Success(stateToApply) => - context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) - val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) + case Success(stateToApply: UtxoState) => + context.parent ! RollbackSucceed(branchingPointOpt) + val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { - case (u, modToApply) => + case (u: UpdateInformation, modToApply: PersistentModifier) => val saveRootNodesFlag: Boolean = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { case Right(stateAfterApply) => - influxRef.foreach( - ref => - modToApply match { - case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) - case _ => - } - ) + modToApply match { + case b: Block if history.isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) + case _ => + } val newHis: History = history.reportModifierIsValid(modToApply) context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) modToApply match { @@ -271,27 +272,23 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! SemanticallySuccessfulModifier(modToApply) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) )) newHis.isFullChainSynced = true - influxRef.foreach { ref => - logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") - ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) - val isBlock: Boolean = modToApply match { - case _: Block => true - case _: Payload => true - case _ => false - } - if (isBlock) ref ! ModifierAppendedToState(success = true) - } - UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) - case Left(e) => + context.parent ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) + if (modToApply match { + case _: Block => true + case _: Payload => true + case _ => false + }) context.parent ! ModifierAppendedToState(success = true) + UpdateInformation(newHis, stateAfterApply, none, none, u.suffix :+ modToApply) + case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") val (newHis: History, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(modToApply) - context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) + context.parent ! SemanticallyFailedModification(modToApply, e) UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) } else u } @@ -302,18 +299,18 @@ class NodeViewHolder( case None => (uf.history, uf.state, uf.suffix) } case Failure(e) => - context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) + context.parent ! RollbackFailed(branchingPointOpt) EncryApp.forceStopApplication(500, s"Rollback failed: $e") } } - def applyModfier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + def applyModifier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = if (!nodeView.history.isModifierDefined(modifier.id)) { logger.debug( s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." ) val startApplicationToTheHistory: Long = System.currentTimeMillis() - influxRef.foreach(_ ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis())) + context.parent ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis()) nodeView.history.append(modifier) match { case Right((historyBeforeStUpdate, progressInfo)) => logger.info( @@ -321,13 +318,11 @@ class NodeViewHolder( s"Time of applying is: ${(System.currentTimeMillis() - startApplicationToTheHistory) / 1000}s." ) if (modifier.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref: ActorRef => - ref ! EndOfApplyingModifier(modifier.id) - ref ! ModifierAppendedToHistory(modifier match { - case _: Header => true - case _: Payload => false - }, success = true) - } + context.parent ! EndOfApplyingModifier(modifier.id) + context.parent ! ModifierAppendedToHistory(modifier match { + case _: Header => true + case _: Payload => false + }, success = true) if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && modifier.modifierTypeId == Payload.modifierTypeId && historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { @@ -353,16 +348,13 @@ class NodeViewHolder( val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced - influxRef.foreach(_ ! StateUpdating(System.currentTimeMillis() - startPoint)) + context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) sendUpdatedInfoToMemoryPool(progressInfo.toRemove) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") - for { - ref <- influxRef - header <- newHistory.getBestHeader - } yield ref ! BestHeaderInChain(header) + newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) if (newHistory.isFullChainSynced) { logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() @@ -371,19 +363,18 @@ class NodeViewHolder( updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") - influxRef.foreach(_ ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height)) + context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + context.parent ! SemanticallySuccessfulModifier(modifier) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } - case Left(e) => + case Left(e: Throwable) => logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") - context.system.eventStream - .publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage)))) + context.parent ! SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage))) } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { val rolledBackTxs: IndexedSeq[Transaction] = toRemove @@ -411,7 +402,7 @@ class NodeViewHolder( NodeView(history, state, wallet) } - def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = + def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) try { val stateDir: File = UtxoState.getStateDir(settings) @@ -436,12 +427,12 @@ class NodeViewHolder( new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) genesisState.some } else { - None + none } def getRecreatedState( - version: Option[VersionTag] = None, - digest: Option[ADDigest] = None, + version: Option[VersionTag] = none, + digest: Option[ADDigest] = none, influxRef: Option[ActorRef] ): UtxoState = { val dir: File = UtxoState.getStateDir(settings) @@ -511,7 +502,7 @@ object NodeViewHolder { final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierId: ModifierId, - previousModifier: Option[ModifierId] = None + previousModifier: Option[ModifierId] = none ) extends NodeViewHolderEvent final case class UpdateInformation( diff --git a/src/main/scala/encry/stats/StatsSender.scala b/src/main/scala/encry/stats/StatsSender.scala index f14af85b0b..deae07515f 100644 --- a/src/main/scala/encry/stats/StatsSender.scala +++ b/src/main/scala/encry/stats/StatsSender.scala @@ -147,16 +147,17 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe } object StatsSender { - final case class BestHeaderInChain(bestHeader: Header) extends AnyVal - final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) - final case class TransactionsInBlock(txsNum: Int) extends AnyVal - final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) - final case class ModifierAppendedToState(success: Boolean) extends AnyVal + sealed trait StatsSenderMessage + final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage + final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage + final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage + final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) extends StatsSenderMessage + final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal - final case class EndOfApplyingModifier(modifierId: ModifierId) extends AnyVal - final case class StateUpdating(time: Long) extends AnyVal + final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage + final case class StateUpdating(time: Long) extends StatsSenderMessage final case class SleepTime(time: Long) extends AnyVal - final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) + final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) extends StatsSenderMessage final case class MiningEnd(blockHeader: Header, workerIdx: Int, workersQty: Int) final case class MiningTime(time: Long) extends AnyVal final case class SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) From d7d88c816b525707f0ba806675516e22734700f1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:57:30 +0300 Subject: [PATCH 019/119] more nvh improvements --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 3e3517b746..ea83e0e8f4 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -82,4 +82,6 @@ class IntermediaryNVH( } } -object IntermediaryNVH {} +object IntermediaryNVH { + +} From 35e695b1ef92f6d069c6d112bbb6e1c149dc4091 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:11:38 +0300 Subject: [PATCH 020/119] changed incoming ModifierFromNetwork message --- src/main/scala/encry/network/PeersKeeper.scala | 2 +- src/main/scala/encry/nvg/IntermediaryNVH.scala | 15 ++++++--------- src/main/scala/encry/nvg/ModifiersValidator.scala | 5 +++-- src/main/scala/encry/view/NodeViewHolder.scala | 5 +++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 7a421d15c7..a6e380fc9e 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -339,7 +339,7 @@ object PeersKeeper { final case class UpdatedPeersCollection(peers: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)]) - final case class BanPeer(peer: ConnectedPeer, reason: BanReason) + final case class BanPeer(peer: InetSocketAddress, reason: BanReason) final case class BanPeerFromAPI(peer: InetSocketAddress, reason: BanReason) extends PeerCommandHelper diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index ea83e0e8f4..53a0907504 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,6 +8,7 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ RollbackFailed, RollbackSucceed, @@ -28,6 +29,7 @@ import encry.view.fast.sync.SnapshotHolder.{ } import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( @@ -52,12 +54,9 @@ class IntermediaryNVH( var historyReader: HistoryReader = HistoryReader.empty override def receive: Receive = { - case ModifiersForValidating(remote, typeId, modifiers) => - logger.info(s"Got ${modifiers.size} modifiers of type $typeId for validation.") - modifiers.foreach { - case (id: ModifierId, bytes: Array[Byte]) => - modifiersValidatorRouter ! ModifierForValidation(historyReader, id, typeId, bytes, remote) - } + case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => + logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") + modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader @@ -82,6 +81,4 @@ class IntermediaryNVH( } } -object IntermediaryNVH { - -} +object IntermediaryNVH {} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 48ce5b076e..215f1ec2a2 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -1,5 +1,7 @@ package encry.nvg +import java.net.InetSocketAddress + import HeaderProto.HeaderProtoMessage import PayloadProto.PayloadProtoMessage import akka.actor.{ Actor, ActorRef, Props } @@ -13,7 +15,6 @@ import encry.network.BlackList.BanReason.{ SyntacticallyInvalidPersistentModifier } import encry.network.DownloadedModifiersValidator.InvalidModifier -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings @@ -96,7 +97,7 @@ object ModifiersValidator { modifierId: ModifierId, modifierTypeId: ModifierTypeId, modifierBytes: Array[Byte], - remote: ConnectedPeer + remote: InetSocketAddress ) final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 514fb93335..67671fb731 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,12 +1,13 @@ package encry.view -import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } -import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } +import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} +import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.NodeViewHolder._ import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction From f50ae42c1865b15a65bea3ca443d63efc8964af7 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 17:33:08 +0300 Subject: [PATCH 021/119] add dm --- src/main/scala/encry/network/DM.scala | 48 ++++++++++++++++++- .../scala/encry/network/NetworkRouter.scala | 2 +- src/main/scala/encry/network/PK.scala | 16 ++++++- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2133fe0742..f92510536b 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,11 +1,55 @@ package encry.network +import java.net.InetSocketAddress + import akka.actor.{Actor, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.DM.{AwaitingRequest, RequestSent} +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkRouter.ModifierFromNetwork +import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.settings.NetworkSettings +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + +import scala.collection.mutable + +case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { + + import context.dispatcher + + type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte -class DM extends Actor { - override def receive: Receive = ??? + var expectedModifiers: Set[ModifierIdAsKey] = Set.empty + var receivedModifier: Set[ModifierIdAsKey] = Set.empty + + override def receive: Receive = { + case RequestSent(peer, modTypeId, modId) => + expectedModifiers += toKey(modId) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, 1) + ) + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => + if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer, modTypeId, List(modId)) + case AwaitingRequest(peer, _, modId, _) => + logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") + case ModifierFromNetwork(source, modTypeId, modId, modBytes) => + if (expectedModifiers.contains(toKey(modId))) { + expectedModifiers -= toKey(modId) + receivedModifier += toKey(modId) + context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) + } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") + case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) + } + + def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) } object DM { + + case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) + case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) + + def props(): Props = ??? } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index df413f773b..1b6ab8e4cb 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -53,7 +53,7 @@ class NetworkRouter(settings: NetworkSettings, logger.debug(s"Got ${message.messageName} on the NetworkController.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) case MessageFromNetwork(message, Some(remote)) => - peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) + peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index d0d13ef2cb..fd67067aed 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -5,7 +5,9 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} @@ -30,7 +32,7 @@ class PK(networkSettings: NetworkSettings, var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - override def receive: Receive = { + override def receive: Receive = banPeersLogic orElse { case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + @@ -60,7 +62,6 @@ class PK(networkSettings: NetworkSettings, case NewConnection(remote, _) => logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + s"bigger than possible or isSelf: ${isSelf(remote)}.") - case HandshakedDone(connectedPeer) => logger.info(s"Peers keeper got approvement about finishing a handshake." + s" Initializing new peer: ${connectedPeer.socketAddress}") @@ -87,6 +88,17 @@ class PK(networkSettings: NetworkSettings, } } + def banPeersLogic: Receive = { + case BanPeer(peer, reason) => + logger.info(s"Banning peer: ${peer} for $reason.") + blackList = blackList.banPeer(reason, peer.getAddress) + connectedPeers.getAll.find(_._1 == peer).map(_._2.connectedPeer.handlerRef ! CloseConnection) + + case BanPeerFromAPI(peer, reason) => + logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") + blackList = blackList.banPeer(reason, peer.getAddress) + } + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || From 06503d4c42131c6d7a82426186bc6e2d9df6c9a0 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:36:14 +0300 Subject: [PATCH 022/119] added processing of sync info and inv messages --- .../encry/network/NetworkController.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 18 +---- .../scala/encry/nvg/IntermediaryNVH.scala | 9 ++- .../encry/nvg/NetworkMessagesProcessor.scala | 65 ++++++++++++++++++- src/main/scala/encry/nvg/NodeViewHolder.scala | 6 +- .../scala/encry/view/NodeViewHolder.scala | 2 +- .../encry/view/history/HistoryReader.scala | 22 +++++++ 7 files changed, 99 insertions(+), 25 deletions(-) diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 92e43451d4..0213394830 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -138,7 +138,7 @@ object NetworkController { object ReceivableMessages { - case class DataFromPeer(message: NetworkMessage, source: ConnectedPeer) + case class DataFromPeer(message: NetworkMessage, source: InetSocketAddress) case class RegisterMessagesHandler(types: Seq[(Byte, String)], handler: ActorRef) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 931768507c..c3e37bf98d 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -113,17 +113,7 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], case _ => //Do nothing } case DataFromPeer(message, remote) => message match { - case SyncInfoNetworkMessage(syncInfo) => Option(history) match { - case Some(historyReader) => - val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) - val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) - logger.info(s"Comparison with $remote having starting points ${idsToString(syncInfo.startingPoints)}. " + - s"Comparison result is $comparison. Sending extension of length ${ext.length}.") - if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") - deliveryManager ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) - peersKeeper ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) - case _ => - } + case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap(id => modifiersRequestCache @@ -170,9 +160,7 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !history.isFullChainSynced => - logger.info(s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + - s"But full chain is not synced. Ignore them.") + case InvNetworkMessage(invData) => logger.debug(s"Got inv message on NodeViewSynchronizer from ${remote.socketAddress} with modifiers of type:" + s" $invData._1. Size of inv is: ${invData._2.size}. Sending CompareViews to NVH. " + @@ -264,7 +252,7 @@ object NodeViewSynchronizer { case object SendLocalSyncInfo - final case class OtherNodeSyncingStatus(remote: ConnectedPeer, + final case class OtherNodeSyncingStatus(remote: InetSocketAddress, status: encry.consensus.HistoryConsensus.HistoryComparisonResult, extension: Option[Seq[(ModifierTypeId, ModifierId)]]) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 53a0907504..1f62613ee5 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -6,10 +6,12 @@ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +import encry.network.DownloadedModifiersValidator.InvalidModifier +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + OtherNodeSyncingStatus, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, @@ -30,7 +32,6 @@ import encry.view.fast.sync.SnapshotHolder.{ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( settings: EncryAppSettings, @@ -65,6 +66,8 @@ class IntermediaryNVH( case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg case msg @ FastSyncDone => networkMessagesProcessor ! msg case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg @@ -73,7 +76,7 @@ class IntermediaryNVH( case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => //+ to stats sender + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 6cb0c78b59..f5038bf53d 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,9 +1,70 @@ package encry.nvg import akka.actor.{ Actor, Props } +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.settings.EncryAppSettings +import encry.utils.Utils.idsToString +import encry.view.ModifiersCache +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + +import scala.collection.Seq + +class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + + var historyReader: HistoryReader = HistoryReader.empty + + override def receive: Receive = { + case DataFromPeer(message, remote) => + message match { + case SyncInfoNetworkMessage(syncInfo: SyncInfo) => + val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) + logger.info( + s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + + s"Comparison result is $comparison. Sending extension of length ${ext.length}." + ) + if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") + context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) + + case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => + logger.info( + s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + + s"But full chain is not synced. Ignore them." + ) + + case InvNetworkMessage(invData) => + val startTime: Long = System.currentTimeMillis() + val ids: Seq[ModifierId] = invData._2.filterNot( + (mid: ModifierId) => + historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) + ) + logger.info( + s"Got inv message from $remote. Type of nested ids is: ${invData._1}. " + + s"Ids for request are: ${ids.map(Algos.encode).mkString(",")}." + ) + if (ids.nonEmpty && + (invData._1 == Header.modifierTypeId || + (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) + sender() ! RequestFromLocal(remote, invData._1, ids.toList) + logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") -class NetworkMessagesProcessor extends Actor { - override def receive: Receive = ??? + case RequestModifiersNetworkMessage(data) => + } + } } object NetworkMessagesProcessor { diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index da205eb964..1250d39c9e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -73,7 +73,7 @@ class NodeViewHolder( override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - val wrappedKey: mutable.WrappedArray.ofByte = toKey(modifier.id) + val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) @@ -374,8 +374,6 @@ class NodeViewHolder( } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) @@ -490,6 +488,8 @@ class NodeViewHolder( object NodeViewHolder { + def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 67671fb731..9e3ead575b 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -7,7 +7,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} import encry.view.NodeViewHolder._ import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 7388d4af38..78e3ff6fb9 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,17 +1,39 @@ package encry.view.history +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + trait HistoryReader { def getBestHeaderHeight: Int + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] + + def compare(si: SyncInfo): HistoryComparisonResult + + var isFullChainSynced: Boolean + + var isHeadersChainSyncedVar: Boolean = false + + def isModifierDefined(id: ModifierId): Boolean + } object HistoryReader { def empty: HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = true def getBestHeaderHeight = 0 + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty + var isFullChainSynced: Boolean = true + def compare(si: SyncInfo): HistoryComparisonResult = Older } def apply(): HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = true def getBestHeaderHeight = 1 + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty + def compare(si: SyncInfo): HistoryComparisonResult = Older + var isFullChainSynced: Boolean = true } } \ No newline at end of file From f37daf885a4b33592a8b1d4b8430b02458d6ea3b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:40:37 +0300 Subject: [PATCH 023/119] code cleaned up + added SSM/SFM to the network --- .../encry/network/NodeViewSynchronizer.scala | 6 ----- .../scala/encry/nvg/IntermediaryNVH.scala | 2 ++ .../scala/encry/view/NodeViewHolder.scala | 25 ------------------- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index c3e37bf98d..50cb8e865c 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -161,12 +161,6 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - case InvNetworkMessage(invData) => - logger.debug(s"Got inv message on NodeViewSynchronizer from ${remote.socketAddress} with modifiers of type:" + - s" $invData._1. Size of inv is: ${invData._2.size}. Sending CompareViews to NVH. " + - s"\nModifiers in inv message are: ${invData._2.map(Algos.encode).mkString(",")}") - nodeViewHolderRef ! CompareViews(remote, invData._1, invData._2) - case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") } case msg@RequestPeersForFirstSyncInfo => diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 1f62613ee5..a1a2989e44 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -80,7 +80,9 @@ class IntermediaryNVH( case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + networkMessagesProcessor ! msg case msg @ SemanticallyFailedModification(_, _) => + networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9e3ead575b..41fd1d535f 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -4,14 +4,10 @@ import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} import encry.view.NodeViewHolder._ -import org.encryfoundation.common.modifiers.history._ -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.Seq @@ -31,31 +27,10 @@ class NodeViewHolder(memoryPoolRef: ActorRef, case resultFuture: Future[_] => resultFuture.pipeTo(sender()) case result => sender() ! result } - case GetNodeViewChanges(history, state, _) => if (history) sender() ! ChangedHistory(nodeView.history) if (state) sender() ! ChangedState(nodeView.state) - case CompareViews(peer, modifierTypeId, modifierIds) => - logger.info(s"Start processing CompareViews message on NVH.") - val startTime = System.currentTimeMillis() - val ids: Seq[ModifierId] = modifierTypeId match { - case _ => - modifierIds - .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) - } - if (modifierTypeId != Transaction.modifierTypeId) - logger.debug( - s"Got compare view message on NVH from ${peer.socketAddress}." + - s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + - s" Sending RequestFromLocal with ids to $sender." + - s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}." - ) - if (ids.nonEmpty && (modifierTypeId == Header.modifierTypeId || (nodeView.history.isHeadersChainSynced && modifierTypeId == Payload.modifierTypeId))) - sender() ! RequestFromLocal(peer, modifierTypeId, ids) - logger.debug( - s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}" - ) case SemanticallySuccessfulModifier(_) => case msg => logger.error(s"Got strange message on nvh: $msg") } From 79e1bd4e1424bd7976ea06173abf6b7e77f48a80 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 18:04:40 +0300 Subject: [PATCH 024/119] add handler for modifierNetMsg --- src/main/scala/encry/network/DM.scala | 3 ++- src/main/scala/encry/network/NetworkRouter.scala | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index f92510536b..deaebb9cac 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -7,7 +7,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkRouter.ModifierFromNetwork -import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -40,6 +40,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) + case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 1b6ab8e4cb..f815b1db88 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,6 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} @@ -24,6 +25,7 @@ class NetworkRouter(settings: NetworkSettings, import context.system var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty + var handlerForMods: ActorRef = ActorRef.noSender IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) @@ -37,6 +39,7 @@ class NetworkRouter(settings: NetworkSettings, val ids = types.map(_._1) messagesHandlers += (ids -> handler) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") + case RegisterForModsHandling => handlerForMods = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") } @@ -55,6 +58,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: ModifierFromNetwork => handlerForMods ! msg case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } @@ -94,7 +98,7 @@ class NetworkRouter(settings: NetworkSettings, mH: Map[Seq[Byte], ActorRef]): Unit = mH.find(_._1.contains(messageId)).map(_._2) match { case Some(handler) => - handler ! DataFromPeer(message, remote) + handler ! DataFromPeer(message, remote.socketAddress) logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") case None => logger.info("No handlers found for message: " + message.messageName) } @@ -107,5 +111,7 @@ object NetworkRouter { modId: ModifierId, modBytes: Array[Byte]) + case object RegisterForModsHandling + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From fef45611d601496bfc3e69840edef820aabc2442 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 12:06:47 +0300 Subject: [PATCH 025/119] network messages improved some additions --- .../encry/api/http/DataHolderForApi.scala | 4 +- .../cli/commands/CreateAccountFromSeed.scala | 1 - src/main/scala/encry/local/miner/Miner.scala | 4 +- .../scala/encry/network/DeliveryManager.scala | 8 +- .../DownloadedModifiersValidator.scala | 1 - .../encry/network/NodeViewSynchronizer.scala | 28 ---- .../scala/encry/nvg/IntermediaryNVH.scala | 28 +++- .../encry/nvg/NetworkMessagesProcessor.scala | 27 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 + .../scala/encry/view/NodeViewHolder.scala | 28 +--- .../encry/view/fast/sync/SnapshotHolder.scala | 154 +++++++++--------- .../encry/view/history/HistoryReader.scala | 4 + 12 files changed, 143 insertions(+), 145 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 72adcec939..7faaf529c0 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -22,7 +22,7 @@ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} -import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.{ GetDataFromCurrentView} import encry.view.history.History import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet @@ -238,7 +238,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) sender() ! history.toList.flatMap(_.headerIdsAtHeight(i).map(Algos.encode)) case CreateAccountManagerFromSeedHelper(seed) => - (nvhRef ? CreateAccountManagerFromSeed(seed)).mapTo[Either[String, EncryWallet]].pipeTo(sender()) + //(nvhRef ? CreateAccountManagerFromSeed(seed)).mapTo[Either[String, EncryWallet]].pipeTo(sender()) case GetAllInfoHelper => diff --git a/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala b/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala index 110580e90a..4a566af27c 100644 --- a/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala +++ b/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala @@ -8,7 +8,6 @@ import encry.settings.EncryAppSettings import encry.EncryApp._ import encry.api.http.DataHolderForApi.CreateAccountManagerFromSeedHelper import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.ReceivableMessages.CreateAccountManagerFromSeed import encry.view.wallet.EncryWallet import scala.concurrent.Future diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 68ca34bbe3..9bc8bde878 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -19,7 +19,7 @@ import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView, LocallyGeneratedModifier} +import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView} import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -111,7 +111,7 @@ class Miner(dataHolder: ActorRef, s" from worker $workerIdx with nonce: ${block.header.nonce}.") logger.debug(s"Set previousSelfMinedBlockId: ${Algos.encode(block.id)}") killAllWorkers() - context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) + //context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) if (settings.influxDB.isDefined) { context.actorSelection("/user/statsSender") ! MiningEnd(block.header, workerIdx, context.children.size) context.actorSelection("/user/statsSender") ! MiningTime(System.currentTimeMillis() - startTime) diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index a1f70e1041..c5219ba652 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -1,6 +1,7 @@ package encry.network import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, ActorSystem, Cancellable, PoisonPill, Props} import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ @@ -12,6 +13,7 @@ import encry.network.PeerConnectionHandler._ import encry.stats.StatsSender.{GetModifiers, SendDownloadRequest, SerializedModifierFromNetwork} import encry.view.history.History import encry.settings.EncryAppSettings + import scala.concurrent.duration._ import scala.collection.immutable.HashSet import scala.collection.{IndexedSeq, mutable} @@ -19,18 +21,20 @@ import scala.util.Random import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.view.NodeViewHolder.DownloadRequest -import encry.view.mempool.MemoryPool.{ StartTransactionsValidation, StopTransactionsValidation } +import encry.nvg.NodeViewHolder.DownloadRequest +import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.ExecutionContextExecutor class DeliveryManager(influxRef: Option[ActorRef], diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index bfeb7c691b..70999f12b6 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -13,7 +13,6 @@ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.History import encry.view.mempool.MemoryPool.NewTransaction import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 50cb8e865c..3f11d22ae2 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -32,7 +32,6 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.view.NodeViewHolder.DownloadRequest import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} @@ -126,32 +125,8 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], if (unrequestedModifiers.nonEmpty) typeId match { case Transaction.modifierTypeId => memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) - case Payload.modifierTypeId => - getModsForRemote(unrequestedModifiers).foreach(_.foreach { - case (id, bytes) => - remote.handlerRef ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) - }) - case tId => getModsForRemote(unrequestedModifiers).foreach { modifiers => - modifiers.foreach(k => - logger.debug(s"Response to ${remote.socketAddress} header ${ - Try(HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(k._2))) - }") - ) - remote.handlerRef ! ModifiersNetworkMessage(tId -> modifiers) - } } - def getModsForRemote(ids: Seq[ModifierId]): Option[Map[ModifierId, Array[Byte]]] = Option(history) - .map { historyStorage => - val modifiers: Map[ModifierId, Array[Byte]] = unrequestedModifiers - .view - .map(id => id -> historyStorage.modifierBytesById(id)) - .collect { case (id, mod) if mod.isDefined => id -> mod.get} - .toMap - logger.debug(s"Send response to $remote with ${modifiers.size} modifiers of type $typeId") - modifiers - } - case RequestModifiersNetworkMessage(requestedIds) => logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") @@ -167,8 +142,6 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + s" peers for first sync info message. Resending $msg to peers keeper.") peersKeeper ! msg - case msg@RequestFromLocal(_, _, _) => deliveryManager ! msg - case msg@DownloadRequest(_, _, _) => deliveryManager ! msg case msg@UpdatedPeersCollection(_) => deliveryManager ! msg case msg@PeersForSyncInfo(_) => logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") @@ -294,7 +267,6 @@ object NodeViewSynchronizer { class NodeViewSynchronizerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case RequestFromLocal(_, _, _) => 0 case DataFromPeer(msg, _) => msg match { case SyncInfoNetworkMessage(_) => 1 diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index a1a2989e44..2938226069 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,7 +8,7 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, @@ -31,6 +31,12 @@ import encry.view.fast.sync.SnapshotHolder.{ } import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -41,8 +47,19 @@ class IntermediaryNVH( ) extends Actor with StrictLogging { + intermediaryNetwork ! RegisterMessagesHandler( + Seq( + InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", + RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", + SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" + ), + self + ) + + //todo add registration for ModifierFromNetwork msg + val networkMessagesProcessor: ActorRef = - context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") + context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") val nodeViewHolder: ActorRef = context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = @@ -68,6 +85,7 @@ class IntermediaryNVH( case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg + case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg @@ -79,10 +97,8 @@ class IntermediaryNVH( case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => - case msg @ SemanticallySuccessfulModifier(_) => - networkMessagesProcessor ! msg - case msg @ SemanticallyFailedModification(_, _) => - networkMessagesProcessor ! msg + case msg @ SemanticallySuccessfulModifier(_) => networkMessagesProcessor ! msg + case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index f5038bf53d..ca48059fb6 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -11,15 +11,15 @@ import encry.utils.Utils.idsToString import encry.view.ModifiersCache import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.history.{ Header, Payload } -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, + ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.Seq @@ -27,17 +27,20 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St var historyReader: HistoryReader = HistoryReader.empty + //todo last modifiers cache? + override def receive: Receive = { case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + val ext: Seq[ModifierId] = + historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + s"Comparison result is $comparison. Sending extension of length ${ext.length}." ) - if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") + if (!(ext.nonEmpty || comparison != Younger)) logger.info("Extension is empty while comparison is younger") context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => @@ -62,11 +65,23 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St sender() ! RequestFromLocal(remote, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - case RequestModifiersNetworkMessage(data) => + case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => + getModsForRemote(requestedIds.toList, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + } + case RequestModifiersNetworkMessage((typeId, requestedIds)) => + context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(requestedIds.toList, historyReader)) } } + + def getModsForRemote(ids: List[ModifierId], reader: HistoryReader): Map[ModifierId, Array[Byte]] = + ids.view + .map(id => id -> reader.modifierBytesById(id)) + .collect { case (id, mod) if mod.isDefined => id -> mod.get } + .toMap } object NetworkMessagesProcessor { - def props: Props = Props(new NetworkMessagesProcessor) + def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 1250d39c9e..e602315275 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -378,6 +378,7 @@ class NodeViewHolder( val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq + //todo compare with toApply if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 41fd1d535f..98fdfc965c 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,38 +1,25 @@ package encry.view -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} -import encry.view.NodeViewHolder._ -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.view.NodeViewHolder.ReceivableMessages.CompareViews +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } import scala.collection.Seq -import scala.concurrent.Future class NodeViewHolder(memoryPoolRef: ActorRef, influxRef: Option[ActorRef], dataHolder: ActorRef, encrySettings: EncryAppSettings) extends Actor - with StrictLogging - with AutoCloseable { + with StrictLogging { override def receive: Receive = { - case GetDataFromCurrentView(f) => - f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { - case resultFuture: Future[_] => resultFuture.pipeTo(sender()) - case result => sender() ! result - } - case GetNodeViewChanges(history, state, _) => - if (history) sender() ! ChangedHistory(nodeView.history) - if (state) sender() ! ChangedState(nodeView.state) - - case SemanticallySuccessfulModifier(_) => - case msg => logger.error(s"Got strange message on nvh: $msg") + case _ => } } @@ -41,7 +28,6 @@ object NodeViewHolder { case class CurrentView[HIS, MS, VL](history: HIS, state: MS, vault: VL) - object ReceivableMessages { case class GetNodeViewChanges(history: Boolean, state: Boolean, vault: Boolean) diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index f8b87a4df3..35e3ef98fc 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -88,52 +88,52 @@ class SnapshotHolder(settings: EncryAppSettings, responseTimeout: Option[Cancellable] ): Receive = { case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote.socketAddress} message ${message.NetworkMessageTypeID}.") + logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") message match { case ResponseManifestMessage(manifest) => logger.info( s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotProcessor.validateChunkId(chunk) - processor = snapshotProcessor.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotProcessor] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotProcessor = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks - } +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotProcessor.validateChunkId(chunk) +// processor = snapshotProcessor.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotProcessor] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotProcessor = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote.socketAddress}") + logger.info(s"Received chunk from unexpected peer ${remote}") case _ => } @@ -211,39 +211,39 @@ class SnapshotHolder(settings: EncryAppSettings, context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) case DataFromPeer(message, remote) => - message match { - case ResponseManifestMessage(manifest) => - val isValidManifest: Boolean = - snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) - val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed - if (isValidManifest && canBeProcessed) { - (for { - controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotProcessor.initializeApplicableChunksCache( - history, - snapshotDownloadController.requiredManifestHeight - ) - } yield (controller, processor)) match { - case Left(error) => - nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) - case Right((controller, processor)) => - logger.debug(s"Request manifest message successfully processed.") - responseManifestTimeout.foreach(_.cancel()) - snapshotDownloadController = controller - snapshotProcessor = processor - self ! RequestNextChunks - logger.debug("Manifest processed successfully.") - context.become(fastSyncMod(history, none)) - } - } else if (!isValidManifest) { - logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") - nodeViewSynchronizer ! BanPeer( - remote, - InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") - ) - } else logger.info(s"Doesn't need to process new manifest.") - case _ => - } +// message match { +// case ResponseManifestMessage(manifest) => +// val isValidManifest: Boolean = +// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) +// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed +// if (isValidManifest && canBeProcessed) { +// (for { +// controller <- snapshotDownloadController.processManifest(manifest, remote, history) +// processor <- snapshotProcessor.initializeApplicableChunksCache( +// history, +// snapshotDownloadController.requiredManifestHeight +// ) +// } yield (controller, processor)) match { +// case Left(error) => +// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) +// case Right((controller, processor)) => +// logger.debug(s"Request manifest message successfully processed.") +// responseManifestTimeout.foreach(_.cancel()) +// snapshotDownloadController = controller +// snapshotProcessor = processor +// self ! RequestNextChunks +// logger.debug("Manifest processed successfully.") +// context.become(fastSyncMod(history, none)) +// } +// } else if (!isValidManifest) { +// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") +// nodeViewSynchronizer ! BanPeer( +// remote, +// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") +// ) +// } else logger.info(s"Doesn't need to process new manifest.") +// case _ => +// } case msg @ RequiredManifestHeightAndId(_, _) => self ! msg @@ -278,19 +278,21 @@ class SnapshotHolder(settings: EncryAppSettings, if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => snapshotProcessor.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") - remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) + //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } case RequestManifestMessage(manifest) => logger.debug(s"Got request for manifest with ${Algos.encode(manifest)}") - case RequestChunkMessage(chunkId) if requestsProcessor.canProcessRequest(remote) => + case RequestChunkMessage(chunkId) + //if requestsProcessor.canProcessRequest(remote) + => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) - remote.handlerRef ! networkMessage + //remote.handlerRef ! networkMessage } - requestsProcessor = requestsProcessor.processRequest(remote) + //requestsProcessor = requestsProcessor.processRequest(remote) case RequestChunkMessage(_) => case _ => } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 78e3ff6fb9..54eefce65b 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -18,6 +18,8 @@ trait HistoryReader { def isModifierDefined(id: ModifierId): Boolean + def modifierBytesById(id: ModifierId): Option[Array[Byte]] + } object HistoryReader { @@ -27,6 +29,7 @@ object HistoryReader { def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty var isFullChainSynced: Boolean = true def compare(si: SyncInfo): HistoryComparisonResult = Older + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None } def apply(): HistoryReader = new HistoryReader { @@ -35,5 +38,6 @@ object HistoryReader { def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty def compare(si: SyncInfo): HistoryComparisonResult = Older var isFullChainSynced: Boolean = true + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None } } \ No newline at end of file From b52a1617642e54383aaf299f367b80066c327b8d Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:35:16 +0300 Subject: [PATCH 026/119] added RequestModifiersNetworkMessage processing --- .../scala/encry/nvg/IntermediaryNVH.scala | 43 +++++++++-------- .../encry/nvg/NetworkMessagesProcessor.scala | 48 +++++++++++++++---- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 2938226069..10eaad95a9 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -17,7 +17,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ SemanticallyFailedModification, SemanticallySuccessfulModifier } -import encry.network.PeersKeeper.BanPeer +import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings @@ -79,25 +79,28 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg - case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => - case msg @ SemanticallySuccessfulModifier(_) => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg + case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg + case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => + case msg @ SemanticallySuccessfulModifier(_) => + networkMessagesProcessor ! msg + networkMessagesProcessor ! msg case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index ca48059fb6..8f351f5282 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -3,14 +3,18 @@ package encry.nvg import akka.actor.{ Actor, Props } import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.network.Broadcast import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.network.PeersKeeper.SendToNetwork import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.ModifiersCache import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, ModifiersNetworkMessage, @@ -21,15 +25,25 @@ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import scala.collection.Seq - class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { var historyReader: HistoryReader = HistoryReader.empty - //todo last modifiers cache? + var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty override def receive: Receive = { + case SemanticallySuccessfulModifier(mod) => + mod match { + case block: Block if historyReader.isFullChainSynced => + List(block.header, block.payload).foreach { mod: PersistentModifier => + logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") + context.parent ! SendToNetwork(InvNetworkMessage(mod.modifierTypeId -> Seq(mod.id)), Broadcast) + } + modifiersRequestCache = Map( + block.encodedId -> toProto(block.header), + block.payload.encodedId -> toProto(block.payload) + ) + } case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => @@ -66,12 +80,26 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => - getModsForRemote(requestedIds.toList, historyReader).foreach { - case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds + .flatMap( + (id: ModifierId) => + modifiersRequestCache + .get(Algos.encode(id)) + .map(id -> _) + ) + .toMap + if (modifiersFromCache.nonEmpty) context.parent ! ModifiersNetworkMessage(typeId -> modifiersFromCache) + val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList + + typeId match { + case h if h == Header.modifierTypeId => + context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(unrequestedModifiers, historyReader)) + case _ => + getModsForRemote(unrequestedModifiers, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + } } - case RequestModifiersNetworkMessage((typeId, requestedIds)) => - context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(requestedIds.toList, historyReader)) } } From 9e68e7b25ec938a839556ea091009dd0da86350f Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:52:27 +0300 Subject: [PATCH 027/119] added check payloads to download functionality --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 ++ .../encry/nvg/NetworkMessagesProcessor.scala | 24 +++++++++++++++++-- .../encry/view/history/HistoryReader.scala | 6 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 10eaad95a9..2c32aa2284 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -19,6 +19,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ } import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage @@ -87,6 +88,7 @@ class IntermediaryNVH( case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg + case msg @ IdsForRequest(_) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8f351f5282..9cb4239a6b 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,14 +1,17 @@ package encry.nvg -import akka.actor.{ Actor, Props } +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import encry.network.Broadcast +import cats.syntax.option._ +import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } import encry.network.PeersKeeper.SendToNetwork +import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.ModifiersCache @@ -27,11 +30,20 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + import context.dispatcher + var historyReader: HistoryReader = HistoryReader.empty var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty - override def receive: Receive = { + override def receive(): Receive = + workingCycle( + context.system.scheduler + .scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) + .some + ) + + def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { case SemanticallySuccessfulModifier(mod) => mod match { case block: Block if historyReader.isFullChainSynced => @@ -101,6 +113,13 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } } } + case CheckPayloadsToDownload => + val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) + logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") + if (newIds.nonEmpty) context.parent ! IdsForRequest(newIds.toList) + val nextCheckModsScheduler: Cancellable = + context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) + context.become(workingCycle(nextCheckModsScheduler.some)) } def getModsForRemote(ids: List[ModifierId], reader: HistoryReader): Map[ModifierId, Array[Byte]] = @@ -111,5 +130,6 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } object NetworkMessagesProcessor { + final case class IdsForRequest(ids: List[ModifierId]) extends AnyVal def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 54eefce65b..3f0640ef69 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -4,6 +4,8 @@ import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.collection.immutable.HashSet + trait HistoryReader { def getBestHeaderHeight: Int @@ -20,6 +22,8 @@ trait HistoryReader { def modifierBytesById(id: ModifierId): Option[Array[Byte]] + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + } object HistoryReader { @@ -30,6 +34,7 @@ object HistoryReader { var isFullChainSynced: Boolean = true def compare(si: SyncInfo): HistoryComparisonResult = Older def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty } def apply(): HistoryReader = new HistoryReader { @@ -39,5 +44,6 @@ object HistoryReader { def compare(si: SyncInfo): HistoryComparisonResult = Older var isFullChainSynced: Boolean = true def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty } } \ No newline at end of file From 47d15113e82aa78f4bc4536b6f19962c8fa41b81 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:56:36 +0300 Subject: [PATCH 028/119] modifiers cache moved to the nvg package --- .../encry/{view => nvg}/ModifiersCache.scala | 6 +++--- .../encry/nvg/NetworkMessagesProcessor.scala | 17 ++++++----------- src/main/scala/encry/nvg/NodeViewHolder.scala | 19 +++++++++---------- 3 files changed, 18 insertions(+), 24 deletions(-) rename src/main/scala/encry/{view => nvg}/ModifiersCache.scala (99%) diff --git a/src/main/scala/encry/view/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala similarity index 99% rename from src/main/scala/encry/view/ModifiersCache.scala rename to src/main/scala/encry/nvg/ModifiersCache.scala index 67ff14c4a7..97e4d5b21c 100644 --- a/src/main/scala/encry/view/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -1,4 +1,4 @@ -package encry.view +package encry.nvg import com.typesafe.scalalogging.StrictLogging import encry.view.history.History @@ -7,11 +7,11 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.annotation.tailrec -import scala.collection.immutable.SortedMap import scala.collection.concurrent.TrieMap +import scala.collection.immutable.SortedMap import scala.collection.mutable -import encry.EncryApp.settings object ModifiersCache extends StrictLogging { diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 9cb4239a6b..7ed1c56f3e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,29 +1,23 @@ package encry.nvg -import akka.actor.{ Actor, Cancellable, Props } +import akka.actor.{Actor, Cancellable, Props} import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} import encry.network.Broadcast import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.network.NodeViewSynchronizer.ReceivableMessages.{OtherNodeSyncingStatus, SemanticallySuccessfulModifier} import encry.network.PeersKeeper.SendToNetwork import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString -import encry.view.ModifiersCache import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - ModifiersNetworkMessage, - RequestModifiersNetworkMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -45,6 +39,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { case SemanticallySuccessfulModifier(mod) => + //todo possible way to call CheckPayloadsToDownload mod match { case block: Block if historyReader.isFullChainSynced => List(block.header, block.payload).foreach { mod: PersistentModifier => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index e602315275..a8f8aae222 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,29 +2,28 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.{DownloadRequest, NodeView, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider -import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -32,14 +31,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, From 55bdbcf83d1261de5cf524998e1c681d182f9962 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 14:02:42 +0300 Subject: [PATCH 029/119] added registration for ModifierFromNetwork msg --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 2c32aa2284..c39df85e62 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -9,7 +9,7 @@ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.ModifierFromNetwork +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, RollbackFailed, @@ -57,7 +57,7 @@ class IntermediaryNVH( self ) - //todo add registration for ModifierFromNetwork msg + intermediaryNetwork ! RegisterForModsHandling val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") From d3df356c68c0dccc6ef5f431e75b6e8137317087 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 14:04:08 +0300 Subject: [PATCH 030/119] minor fix --- .../scala/encry/nvg/IntermediaryNVH.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c39df85e62..d338e100fb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -80,19 +80,19 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg - case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg - case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg - case msg @ IdsForRequest(_) => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => intermediaryNetwork ! msg + case msg @ InvalidModifier(_) => intermediaryNetwork ! msg + case msg @ FastSyncDone => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ ModifiersNetworkMessage(_, _) => intermediaryNetwork ! msg + case msg @ SendToNetwork(_, _) => intermediaryNetwork ! msg + case msg @ IdsForRequest(_) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ HeaderChainIsSynced => intermediaryNetwork ! msg + case msg @ FullBlockChainIsSynced => intermediaryNetwork ! msg //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder @@ -101,9 +101,9 @@ class IntermediaryNVH( case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + intermediaryNetwork ! msg networkMessagesProcessor ! msg - networkMessagesProcessor ! msg - case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg + case msg @ SemanticallyFailedModification(_, _) => intermediaryNetwork ! msg } } From 2f28e798353e2e9682dd55ea9282d940a984f61c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 15:12:23 +0300 Subject: [PATCH 031/119] comment old actors --- src/main/scala/encry/Starter.scala | 25 +- src/main/scala/encry/network/DM.scala | 9 +- .../scala/encry/network/DeliveryManager.scala | 967 +++++++++--------- .../DownloadedModifiersValidator.scala | 110 +- .../scala/encry/network/MessageBuilder.scala | 22 +- .../encry/network/NetworkController.scala | 224 ++-- .../scala/encry/network/NetworkRouter.scala | 16 +- .../encry/network/NodeViewSynchronizer.scala | 374 +++---- src/main/scala/encry/network/PK.scala | 29 +- .../encry/network/PeerConnectionHandler.scala | 2 +- .../scala/encry/network/PeersKeeper.scala | 570 +++++------ .../scala/encry/network/BlackListTests.scala | 194 ++-- .../network/ConnectWithNewPeerTests.scala | 408 ++++---- .../DeliveryManagerTests/DMUtils.scala | 76 +- .../DeliveryManagerPriorityTests.scala | 438 ++++---- ...DeliveryManagerReRequestModifiesSpec.scala | 374 +++---- .../DeliveryManagerRequestModifiesSpec.scala | 634 ++++++------ .../DownloadedModifiersValidatorTests.scala | 406 ++++---- .../HistoryComparisionResultTest.scala | 240 ++--- .../history/ModifiersValidationTest.scala | 90 +- 20 files changed, 2634 insertions(+), 2574 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index a7b8249851..25b5fd71e7 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,10 +1,11 @@ package encry import java.net.InetSocketAddress -import akka.actor.{ Actor, ActorRef } + +import akka.actor.{Actor, ActorRef} import akka.http.scaladsl.Http import cats.Functor -import cats.data.{ NonEmptyChain, Validated } +import cats.data.{NonEmptyChain, Validated} import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -15,19 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{ prompt, StartListening } +import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining -import encry.network.NodeViewSynchronizer +import encry.network.{NetworkRouter, NodeViewSynchronizer} import encry.settings._ import encry.stats.StatsSender -import encry.utils.{ Mnemonic, NetworkTimeProvider } +import encry.utils.{Mnemonic, NetworkTimeProvider} import encry.view.NodeViewHolder import encry.view.mempool.MemoryPool import encry.view.wallet.AccountManager + import scala.concurrent.Future import scala.io.StdIn -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, @@ -422,9 +424,16 @@ class Starter(settings: EncryAppSettings, if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) +// context.system.actorOf( +// NodeViewSynchronizer +// .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) +// .withDispatcher("nvsh-dispatcher"), +// "nodeViewSynchronizer" +// ) + context.system.actorOf( - NodeViewSynchronizer - .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) + NetworkRouter + .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), "nodeViewSynchronizer" ) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index deaebb9cac..60e775f32e 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -6,9 +6,11 @@ import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -23,6 +25,9 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging var expectedModifiers: Set[ModifierIdAsKey] = Set.empty var receivedModifier: Set[ModifierIdAsKey] = Set.empty + override def preStart(): Unit = + context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) + override def receive: Receive = { case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) @@ -50,7 +55,7 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) + case class IsRequested(modifierId: ModifierId) - - def props(): Props = ??? + def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index c5219ba652..3fcd742e84 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -22,6 +22,7 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.ConnectionStatusMessages.ConnectionStopped import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus @@ -36,475 +37,475 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.ExecutionContextExecutor - -class DeliveryManager(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - networkControllerRef: ActorRef, - memoryPoolRef: ActorRef, - nodeViewSync: ActorRef, - downloadedModifiersValidator: ActorRef, - settings: EncryAppSettings) extends Actor with StrictLogging { - - type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte - - implicit val exCon: ExecutionContextExecutor = context.dispatcher - - /** - * Collection with spam modifiers. - * Modifier considered spam if we receive it but it doesn't contain in expected modifiers collection. - */ - var receivedSpamModifiers: Map[ModifierIdAsKey, ConnectedPeer] = Map.empty - /** - * Collection of received modifiers ids. - * Modifier considered received if we sent request for it and received it in special period. - */ - var receivedModifiers: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] - /** - * Collection of expected modifiers ids. - * Modifier considered expected if we sent request for it. - */ - var expectedModifiers: Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = Map.empty - - var expectedTransactions: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] - - var peersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)] = Map.empty - - var priorityCalculator: PrioritiesCalculator = PrioritiesCalculator(settings.network) - - var canProcessTransactions: Boolean = true - - override def preStart(): Unit = { - networkControllerRef ! RegisterMessagesHandler( - Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) - context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) - } - - override def receive: Receive = { - case UpdatedHistory(historyReader) => - logger.debug(s"Got message with history. Starting normal actor's work.") - context.system.scheduler.schedule(0.second, priorityCalculator.updatingStatisticTime) { - val (accumulatedStatistic: Map[InetSocketAddress, PeersPriorityStatus], newStat: PrioritiesCalculator) = - priorityCalculator.accumulatePeersStatistic - priorityCalculator = newStat - context.parent ! AccumulatedPeersStatistic(accumulatedStatistic) - } - val checkModsSch = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)( - self ! CheckPayloadsToDownload - ) - nodeViewSync ! SendLocalSyncInfo - context.become(basicMessageHandler(historyReader, isBlockChainSynced = false, isMining = settings.node.mining, checkModsSch)) - case message => logger.debug(s"Got new message $message while awaiting history.") - } - - def basicMessageHandler(history: History, - isBlockChainSynced: Boolean, - isMining: Boolean, - checkModScheduler: Cancellable): Receive = { - case InvalidModifier(id) => receivedModifiers -= toKey(id) - - case CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) => - checkDelivery(peer, modifierTypeId, modifierId) - - case UpdatedPeersCollection(newPeers) => - logger.info(s"Delivery manager got updated peers collection.") - peersCollection = newPeers - - case ConnectionStopped(peer) => - peersCollection -= peer - logger.info(s"Removed peer: $peer from peers collection on Delivery Manager." + - s" Current peers are: ${peersCollection.mkString(",")}") - - case OtherNodeSyncingStatus(remote, status, extOpt) => - status match { - case Unknown => logger.info("Peer status is still unknown.") - case Younger | Fork if isBlockChainSynced => sendInvData(remote, status, extOpt) - case _ => - } - - case CheckPayloadsToDownload => - val currentQueue: HashSet[ModifierIdAsKey] = - expectedModifiers.flatMap { case (_, modIds) => modIds.keys }.to[HashSet] - logger.debug(s"Current queue: ${currentQueue.map(elem => Algos.encode(elem.toArray)).mkString(",")}") - logger.debug(s"receivedModifiers: ${receivedModifiers.map(id => Algos.encode(id.toArray)).mkString(",")}") - logger.debug(s"Qty to req: ${settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size}") - logger.debug(s"currentQueue.size: ${currentQueue.size}") - logger.debug(s"receivedModifiers.size: ${receivedModifiers.size}") - val newIds: Seq[ModifierId] = - history.payloadsIdsToDownload( - settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size, - currentQueue.map(elem => ModifierId @@ elem.toArray) - ).filterNot(modId => currentQueue.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) - logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) requestDownload(Payload.modifierTypeId, newIds, history, isBlockChainSynced, isMining) - val nextCheckModsScheduler = - context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) - context.become(basicMessageHandler(history, isBlockChainSynced, settings.node.mining, nextCheckModsScheduler)) - - case SemanticallySuccessfulModifier(mod) => - logger.info(s"Got SemanticallySuccessfulModifier with id: ${Algos.encode(mod.id)} of type ${mod.modifierTypeId} on dm") - mod match { - case block: Block => receivedModifiers -= toKey(block.payload.id) - case _ => receivedModifiers -= toKey(mod.id) - } - if (!isBlockChainSynced && expectedModifiers.isEmpty && receivedModifiers.isEmpty) { - checkModScheduler.cancel() - logger.debug(s"SemanticallySuccessfulModifier case, if condition true. Resend CheckPayloadsToDownload to DM") - self ! CheckPayloadsToDownload - } - case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) - - case SyntacticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) - - case SuccessfulTransaction(_) => //do nothing - - case RequestFromLocal(peer, modifierTypeId, modifierIds) => - if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got RequestFromLocal on NVSH from $sender with " + - s"ids of type: $modifierTypeId. Number of ids is: ${modifierIds.size}. Ids: ${modifierIds.map(Algos.encode).mkString(",")}. Sending request from local to DeliveryManager.") - if (modifierIds.nonEmpty) requestModifies(history, peer, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - - case DataFromPeer(message, remote) => message match { - case ModifiersNetworkMessage((typeId, modifiers)) => - logger.debug(s"Received modifiers are: ${modifiers.map(x => Algos.encode(x._1)).mkString(",")}") - influxRef.foreach(_ ! GetModifiers(typeId, modifiers.keys.toSeq)) - for ((id, _) <- modifiers) receive(typeId, id, remote, isBlockChainSynced) - val (spam: Map[ModifierId, Array[Byte]], fm: Map[ModifierId, Array[Byte]]) = modifiers.partition(p => isSpam(p._1)) - if (spam.nonEmpty) { - if (typeId != Transaction.modifierTypeId) - logger.info(s"Spam attempt: peer $remote has sent a non-requested modifiers of type $typeId with ids" + - s": ${spam.keys.map(Algos.encode)}.") - receivedSpamModifiers = Map.empty - } - val filteredModifiers: Map[ModifierId, Array[Byte]] = fm.filterKeys(k => !history.isModifierDefined(k)) - if (typeId != Transaction.modifierTypeId) influxRef - .foreach(ref => (0 to filteredModifiers.size).foreach(_ => ref ! SerializedModifierFromNetwork(typeId))) - //todo check this logic - logger.debug(s"Type of mod: ${typeId}. canProcessTransactions: ${canProcessTransactions}") - if ((typeId == Transaction.modifierTypeId && canProcessTransactions) || (typeId != Transaction.modifierTypeId)) - downloadedModifiersValidator ! ModifiersForValidating(remote, typeId, filteredModifiers) - - case _ => logger.debug(s"DeliveryManager got invalid type of DataFromPeer message!") - } - - case DownloadRequest(modifierTypeId, modifiersId, previousModifier) => //todo check this condition - if (modifierTypeId != Transaction.modifierTypeId) - logger.info(s"DownloadRequest for mod ${Algos.encode(modifiersId)} of type: $modifierTypeId prev mod: " + - s"${previousModifier.map(Algos.encode)}") - requestDownload(modifierTypeId, Seq(modifiersId), history, isBlockChainSynced, isMining) - - case PeersForSyncInfo(peers) => sendSync(history.syncInfo, peers) - - case FullBlockChainIsSynced => context.become(basicMessageHandler(history, isBlockChainSynced = true, isMining, checkModScheduler)) - - case StartMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = true, checkModScheduler)) - - case DisableMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = false, checkModScheduler)) - - case UpdatedHistory(historyReader) => context.become(basicMessageHandler(historyReader, isBlockChainSynced, isMining, checkModScheduler)) - - case StopTransactionsValidation => canProcessTransactions = false - - case StartTransactionsValidation => canProcessTransactions = true - - case message => logger.debug(s"Got strange message $message(${message.getClass}) on DeliveryManager from $sender") - } - - /** - * This function check if modifier has received or not. - * If modifier has transaction type id, it won't be re-asked. - * If we still have no this modifier and number of attempts have no expired, we will re-ask it. - * If we still have no this modifier and number of attempts have expired, we will remove it from expected modifiers collection. - * Otherwise - do nothing. - * - * @param peer - peer, from whom we are expecting modifier - * @param modifierTypeId - type of expected modifier - * @param modifierId - expected modifier id - */ - def checkDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId): Unit = { - val expectedModifiersByPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = - expectedModifiers.getOrElse(peer.socketAddress, Map.empty) - if (modifierTypeId == Transaction.modifierTypeId) - expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, toKey(modifierId), peer.socketAddress) - else expectedModifiersByPeer.find { case (id, (_, _)) => id == toKey(modifierId) } match { - case Some((_, (_, attempts))) if attempts <= settings.network.maxDeliveryChecks => - logger.debug(s"Modifier ${Algos.encode(modifierId)} needed to be requested from $peer!") - reRequestModifier(peer, modifierTypeId, modifierId, expectedModifiersByPeer) - case Some((modId, (_, _))) => - logger.debug(s"Maximum number of attempts has expired. Remove modifier ${Algos.encode(modifierId)} from $peer.") - expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, modId, peer.socketAddress) - case _ => - logger.debug(s"This modifiers ${Algos.encode(modifierId)} is not contained in expectedModifiers collection from $peer.") - } - } - - /** - * If node is not synced, send sync info to random peer, otherwise to all known peers. - * - * @param syncInfo - sync info - */ - def sendSync(syncInfo: SyncInfo, peers: Seq[ConnectedPeer]): Unit = peers.foreach { peer => - logger.info(s"Sending to $peer sync info message.") - peer.handlerRef ! SyncInfoNetworkMessage(syncInfo) - } - - /** - * Send request to 'peer' with modifiers ids of type 'modifierTypeId'. - * We can do this activity only if 'peer' status != Younger. - * If current chain isn't synced and mining is off, we can't request transactions, otherwise can. - * - * We should filter our requesting modifiers to avoid request repeated modifiers. - * - * @param history - current history reader - * @param peer - peer, whom message will be send - * @param mTypeId - modifier type id - * @param modifierIds - modifiers ids - * @param isBlockChainSynced - current block chain status - * @param isMining - current mining status - */ - - def requestModifies(history: History, - peer: ConnectedPeer, - mTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId], - isBlockChainSynced: Boolean, - isMining: Boolean): Unit = { - val firstCondition: Boolean = mTypeId == Transaction.modifierTypeId && isBlockChainSynced && isMining - val secondCondition: Boolean = mTypeId != Transaction.modifierTypeId - val thirdCondition: Boolean = - if (!isBlockChainSynced) peersCollection.get(peer.socketAddress).exists(p => p._2 != Younger) - else peersCollection.contains(peer.socketAddress) - if (mTypeId != Transaction.modifierTypeId) - logger.debug(s"Got requestModifier for modifiers of type: $mTypeId to $peer with modifiers ${modifierIds.size}." + - s" Try to check conditions: $firstCondition -> $secondCondition -> $thirdCondition.") - if ((firstCondition || secondCondition) && thirdCondition) { - val requestedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers - .getOrElse(peer.socketAddress, Map.empty) - - val notYetRequested: Seq[ModifierId] = modifierIds - .filter(id => - !history.isModifierDefined(id) && - !requestedModifiersFromPeer.contains(toKey(id)) && - !receivedModifiers.contains(toKey(id)) - ) - - if (notYetRequested.nonEmpty) { - if (mTypeId != Transaction.modifierTypeId) - logger.debug(s"Send request to ${peer.socketAddress} for ${notYetRequested.size} modifiers of type $mTypeId ") - peer.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> notYetRequested) - priorityCalculator = priorityCalculator.incrementRequestForNModifiers(peer.socketAddress, notYetRequested.size) - if (mTypeId != Transaction.modifierTypeId) { - val requestedModIds: Map[ModifierIdAsKey, (Cancellable, Int)] = - notYetRequested.foldLeft(requestedModifiersFromPeer) { case (rYet, id) => - rYet.updated(toKey(id), - context.system - .scheduler.scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, id)) -> 1) - } - expectedModifiers = expectedModifiers.updated(peer.socketAddress, requestedModIds) - } else expectedTransactions = expectedTransactions ++ modifierIds.map(toKey) - } - } - } - - /** - * Re-ask 'modifierId' from 'peer' if needed. We will do this only if we are expecting these modifier from 'peer' - * and if number of attempts doesn't expired yet. - * This activity will update timer on re-asked modifier. - * - * @param peer - peer, whom message will be send - * @param mTypeId - modifier type id - * @param modId - re-asked modifier id - */ - def reRequestModifier(peer: ConnectedPeer, - mTypeId: ModifierTypeId, - modId: ModifierId, - peerRequests: Map[ModifierIdAsKey, (Cancellable, Int)]): Unit = - peerRequests.get(toKey(modId)) match { - case Some((_, attempts)) => peersCollection.find { case (innerAddr, (_, cResult, _)) => - innerAddr == peer.socketAddress && cResult != Younger - } match { - case Some((_, (cP, _, _))) => - cP.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> Seq(modId)) - logger.debug(s"Re-asked ${peer.socketAddress} and handler: ${peer.handlerRef} for modifier of type: " + - s"$mTypeId with id: ${Algos.encode(modId)}. Attempts: $attempts") - priorityCalculator = priorityCalculator.incrementRequest(peer.socketAddress) - expectedModifiers = expectedModifiers.updated(peer.socketAddress, peerRequests.updated( - toKey(modId), - context.system.scheduler - .scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, modId)) -> (attempts + 1) - )) - case None => - expectedModifiers = clearExpectedModifiersCollection(peerRequests, toKey(modId), peer.socketAddress) - logger.debug(s"Tried to re-ask modifier ${Algos.encode(modId)}, but this id not needed from this peer") - } - case _ => logger.debug(s"There is no such modifier ${Algos.encode(modId)} in expected collection.") - } - - /** - * Check 'expectedModifiers' for awaiting modifier with id 'mId' from 'peer' - * - * @param mId - id of checkable modifier - * @param peer - peer from which we possibly expecting modifier - * @return 'true' if we are expecting this modifier from this peer otherwise 'false' - */ - def isExpecting(mId: ModifierId, modifierTypeId: ModifierTypeId, peer: ConnectedPeer): Boolean = { - if (modifierTypeId != Transaction.modifierTypeId) { - val result: Boolean = expectedModifiers.getOrElse(peer.socketAddress, Map.empty).contains(toKey(mId)) - logger.debug(s"isExpecting -->> modId ${Algos.encode(mId)} --> $result") - result - } else expectedTransactions.contains(toKey(mId)) - } - - /** - * Clear the 'receivedSpamModifiers' collection - * - * @param mIds - sequence of modifiers ids which will be deleted from spam collection - */ - def deleteSpam(mIds: Seq[ModifierId]): Unit = for (id <- mIds) receivedSpamModifiers -= toKey(id) - - /** - * Check receivedSpamModifiers for contains received modifier - * - * @param mId - checkable modifier - * @return 'true' if received modifier is in spam collection otherwise 'false' - */ - def isSpam(mId: ModifierId): Boolean = receivedSpamModifiers.contains(toKey(mId)) - - /** - * Send inv data to the 'peer'. - * - * @param peer - peer whom will send a message - * @param status - current peer's status - * @param dataForInvMessage - data for inv message - */ - def sendInvData(peer: ConnectedPeer, - status: HistoryComparisonResult, - dataForInvMessage: Option[Seq[(ModifierTypeId, ModifierId)]]): Unit = dataForInvMessage match { - case Some(data) => - data.groupBy(_._1).mapValues(_.map(_._2)).foreach { - case (mTid, mods) if mods.size <= settings.network.maxInvObjects => - logger.debug(s"Send to peer $peer inv msg with mods: ${mods.map(Algos.encode).mkString(",")}") - peer.handlerRef ! InvNetworkMessage(mTid -> mods) - case (mTid, mods) => - val modifiers: Seq[ModifierId] = mods.take(settings.network.maxInvObjects) - logger.debug(s"Send to peer $peer dropped inv msg with mods: ${modifiers.map(Algos.encode).mkString(",")}") - peer.handlerRef ! InvNetworkMessage(mTid -> modifiers) - } - case None => logger.info(s"dataForInvMessage is empty for: $peer. Peer's status is: $status.") - } - - /** - * If node is not synced, `requestDownload` sends request for the one peer which will be find by 2 criteria: - * 1) HistoryComparisonResult != Younger. - * 2) Choose random peer with non bad priority. - * Otherwise this function sends requests for all known peers selected by 1-st criterion as above. - * - * If there are no any peers, request won't be sent. - * - * @param modifierTypeId - modifier type id - * @param modifierIds - modifier id - * @param history - current history state - * @param isBlockChainSynced - current block chain status - * @param isMining - current mining status - */ - def requestDownload(modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId], - history: History, - isBlockChainSynced: Boolean, - isMining: Boolean): Unit = - if (!isBlockChainSynced) { - logger.debug(s"requestDownload -> !isBlockChainSynced = true") - val (withBadNodesMap, withoutBadNodesMap) = peersCollection.filter(p => p._2._2 != Younger).partition { - case (_, (_, _, priority)) => priority == BadNode - } - logger.debug(s"withBadNodesMap -> ${withBadNodesMap.keys.mkString(",")}") - logger.debug(s"withoutBadNodesMap -> ${withoutBadNodesMap.keys.mkString(",")}") - val withBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = - withBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq - val withoutBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = - withoutBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq - val resultedPeerCollection = - if (withBadNodes.nonEmpty) withoutBadNodes :+ Random.shuffle(withBadNodes).head - else withoutBadNodes - logger.debug(s"resultedPeerCollection -> $resultedPeerCollection") - logger.debug(s"Block chain is not synced. acceptedPeers: $resultedPeerCollection") - if (resultedPeerCollection.nonEmpty) { - val shuffle: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = Random.shuffle(resultedPeerCollection) - val cP = shuffle.last._1 - influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) - if (modifierTypeId != Transaction.modifierTypeId) - logger.debug(s"requestModifies for peer ${cP.socketAddress} for mods: ${modifierIds.map(Algos.encode).mkString(",")}") - requestModifies(history, cP, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - } else logger.info(s"BlockChain is not synced. There is no nodes, which we can connect with.") - } - else peersCollection.filter(p => p._2._2 != Younger) match { - case coll: Map[_, _] if coll.nonEmpty => - influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) - coll.foreach { case (_, (cp, _, _)) => - if (modifierTypeId != Transaction.modifierTypeId) - logger.info(s"Sent download request to the ${cp.socketAddress} to modifiers of type: $modifierTypeId.") - requestModifies(history, cp, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - } - case _ => logger.info(s"BlockChain is synced. There is no nodes, which we can connect with.") - } - - /** - * Handle received modifier. We will process received modifier only if we are expecting this on. - * - * @param mTid - modifier type id - * @param mId - modifier id - * @param peer - peer who sent modifier - * @param isBlockChainSynced - current chain status - */ - def receive(mTid: ModifierTypeId, - mId: ModifierId, - peer: ConnectedPeer, - isBlockChainSynced: Boolean): Unit = - if (isExpecting(mId, mTid, peer)) { - if (mTid != Transaction.modifierTypeId) { - logger.debug(s"Got new modifier with type $mTid from: ${peer.socketAddress}. with id ${Algos.encode(mId)}") - } - priorityCalculator = priorityCalculator.incrementReceive(peer.socketAddress) - val peerExpectedModifiers: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers - .getOrElse(peer.socketAddress, Map.empty) - peerExpectedModifiers.get(toKey(mId)).foreach(_._1.cancel()) - if (mTid != Transaction.modifierTypeId) receivedModifiers += toKey(mId) - if (mTid != Transaction.modifierTypeId) expectedModifiers = clearExpectedModifiersCollection(peerExpectedModifiers, toKey(mId), peer.socketAddress) - else expectedTransactions = expectedTransactions - toKey(mId) - } else { - receivedSpamModifiers = receivedSpamModifiers - toKey(mId) + (toKey(mId) -> peer) - priorityCalculator = priorityCalculator.decrementRequest(peer.socketAddress) - } - - /** - * Transform modifier id to WrappedArray.ofBytes - * - * @param id - modifier id which will be transform to WrappedArray of bytes. - * @return transformed modifier id - */ - def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) - - /** - * This function gets collection of current expected modifiers from 'peer' and modifier, which - * will be removed from received collection as a parameters. - * If expected modifiers collection will contain other modifiers even after removing, - * this function will return collection of expectedModifiers with updated 'peer' expected collection - * otherwise it will return expectedModifiers collection without 'peer'. - * - * @param expectedModifiersFromPeer - collection of expected modifiers from 'peer' - * @param modifierId - modifier id, which will be removed from 'expectedModifiersFromPeer' - * @param peer - 'peer' from which expected modifiers collection we remove received modifier - * @return - expectedModifiers collection without 'peer' or expectedModifiers with updated 'peer' expected collection - */ - def clearExpectedModifiersCollection(expectedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)], - modifierId: ModifierIdAsKey, - peer: InetSocketAddress): Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = { - val collectionWithoutModId: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiersFromPeer - modifierId - collectionWithoutModId match { - case coll: Map[_, _] if coll.nonEmpty => expectedModifiers.updated(peer, coll) - case _ => expectedModifiers - peer - } - } -} - +// +//class DeliveryManager(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// networkControllerRef: ActorRef, +// memoryPoolRef: ActorRef, +// nodeViewSync: ActorRef, +// downloadedModifiersValidator: ActorRef, +// settings: EncryAppSettings) extends Actor with StrictLogging { +// +// type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte +// +// implicit val exCon: ExecutionContextExecutor = context.dispatcher +// +// /** +// * Collection with spam modifiers. +// * Modifier considered spam if we receive it but it doesn't contain in expected modifiers collection. +// */ +// var receivedSpamModifiers: Map[ModifierIdAsKey, ConnectedPeer] = Map.empty +// /** +// * Collection of received modifiers ids. +// * Modifier considered received if we sent request for it and received it in special period. +// */ +// var receivedModifiers: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] +// /** +// * Collection of expected modifiers ids. +// * Modifier considered expected if we sent request for it. +// */ +// var expectedModifiers: Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = Map.empty +// +// var expectedTransactions: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] +// +// var peersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)] = Map.empty +// +// var priorityCalculator: PrioritiesCalculator = PrioritiesCalculator(settings.network) +// +// var canProcessTransactions: Boolean = true +// +// override def preStart(): Unit = { +// networkControllerRef ! RegisterMessagesHandler( +// Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) +// context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) +// } +// +// override def receive: Receive = { +// case UpdatedHistory(historyReader) => +// logger.debug(s"Got message with history. Starting normal actor's work.") +// context.system.scheduler.schedule(0.second, priorityCalculator.updatingStatisticTime) { +// val (accumulatedStatistic: Map[InetSocketAddress, PeersPriorityStatus], newStat: PrioritiesCalculator) = +// priorityCalculator.accumulatePeersStatistic +// priorityCalculator = newStat +// context.parent ! AccumulatedPeersStatistic(accumulatedStatistic) +// } +// val checkModsSch = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)( +// self ! CheckPayloadsToDownload +// ) +// nodeViewSync ! SendLocalSyncInfo +// context.become(basicMessageHandler(historyReader, isBlockChainSynced = false, isMining = settings.node.mining, checkModsSch)) +// case message => logger.debug(s"Got new message $message while awaiting history.") +// } +// +// def basicMessageHandler(history: History, +// isBlockChainSynced: Boolean, +// isMining: Boolean, +// checkModScheduler: Cancellable): Receive = { +// case InvalidModifier(id) => receivedModifiers -= toKey(id) +// +// case CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) => +// checkDelivery(peer, modifierTypeId, modifierId) +// +// case UpdatedPeersCollection(newPeers) => +// logger.info(s"Delivery manager got updated peers collection.") +// peersCollection = newPeers +// +// case ConnectionStopped(peer) => +// peersCollection -= peer +// logger.info(s"Removed peer: $peer from peers collection on Delivery Manager." + +// s" Current peers are: ${peersCollection.mkString(",")}") +// +// case OtherNodeSyncingStatus(remote, status, extOpt) => +// status match { +// case Unknown => logger.info("Peer status is still unknown.") +// case Younger | Fork if isBlockChainSynced => sendInvData(remote, status, extOpt) +// case _ => +// } +// +// case CheckPayloadsToDownload => +// val currentQueue: HashSet[ModifierIdAsKey] = +// expectedModifiers.flatMap { case (_, modIds) => modIds.keys }.to[HashSet] +// logger.debug(s"Current queue: ${currentQueue.map(elem => Algos.encode(elem.toArray)).mkString(",")}") +// logger.debug(s"receivedModifiers: ${receivedModifiers.map(id => Algos.encode(id.toArray)).mkString(",")}") +// logger.debug(s"Qty to req: ${settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size}") +// logger.debug(s"currentQueue.size: ${currentQueue.size}") +// logger.debug(s"receivedModifiers.size: ${receivedModifiers.size}") +// val newIds: Seq[ModifierId] = +// history.payloadsIdsToDownload( +// settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size, +// currentQueue.map(elem => ModifierId @@ elem.toArray) +// ).filterNot(modId => currentQueue.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) +// logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") +// if (newIds.nonEmpty) requestDownload(Payload.modifierTypeId, newIds, history, isBlockChainSynced, isMining) +// val nextCheckModsScheduler = +// context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) +// context.become(basicMessageHandler(history, isBlockChainSynced, settings.node.mining, nextCheckModsScheduler)) +// +// case SemanticallySuccessfulModifier(mod) => +// logger.info(s"Got SemanticallySuccessfulModifier with id: ${Algos.encode(mod.id)} of type ${mod.modifierTypeId} on dm") +// mod match { +// case block: Block => receivedModifiers -= toKey(block.payload.id) +// case _ => receivedModifiers -= toKey(mod.id) +// } +// if (!isBlockChainSynced && expectedModifiers.isEmpty && receivedModifiers.isEmpty) { +// checkModScheduler.cancel() +// logger.debug(s"SemanticallySuccessfulModifier case, if condition true. Resend CheckPayloadsToDownload to DM") +// self ! CheckPayloadsToDownload +// } +// case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) +// +// case SyntacticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) +// +// case SuccessfulTransaction(_) => //do nothing +// +// case RequestFromLocal(peer, modifierTypeId, modifierIds) => +// if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got RequestFromLocal on NVSH from $sender with " + +// s"ids of type: $modifierTypeId. Number of ids is: ${modifierIds.size}. Ids: ${modifierIds.map(Algos.encode).mkString(",")}. Sending request from local to DeliveryManager.") +// if (modifierIds.nonEmpty) requestModifies(history, peer, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// +// case DataFromPeer(message, remote) => message match { +// case ModifiersNetworkMessage((typeId, modifiers)) => +// logger.debug(s"Received modifiers are: ${modifiers.map(x => Algos.encode(x._1)).mkString(",")}") +// influxRef.foreach(_ ! GetModifiers(typeId, modifiers.keys.toSeq)) +// for ((id, _) <- modifiers) receive(typeId, id, remote, isBlockChainSynced) +// val (spam: Map[ModifierId, Array[Byte]], fm: Map[ModifierId, Array[Byte]]) = modifiers.partition(p => isSpam(p._1)) +// if (spam.nonEmpty) { +// if (typeId != Transaction.modifierTypeId) +// logger.info(s"Spam attempt: peer $remote has sent a non-requested modifiers of type $typeId with ids" + +// s": ${spam.keys.map(Algos.encode)}.") +// receivedSpamModifiers = Map.empty +// } +// val filteredModifiers: Map[ModifierId, Array[Byte]] = fm.filterKeys(k => !history.isModifierDefined(k)) +// if (typeId != Transaction.modifierTypeId) influxRef +// .foreach(ref => (0 to filteredModifiers.size).foreach(_ => ref ! SerializedModifierFromNetwork(typeId))) +// //todo check this logic +// logger.debug(s"Type of mod: ${typeId}. canProcessTransactions: ${canProcessTransactions}") +// if ((typeId == Transaction.modifierTypeId && canProcessTransactions) || (typeId != Transaction.modifierTypeId)) +// downloadedModifiersValidator ! ModifiersForValidating(remote, typeId, filteredModifiers) +// +// case _ => logger.debug(s"DeliveryManager got invalid type of DataFromPeer message!") +// } +// +// case DownloadRequest(modifierTypeId, modifiersId, previousModifier) => //todo check this condition +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.info(s"DownloadRequest for mod ${Algos.encode(modifiersId)} of type: $modifierTypeId prev mod: " + +// s"${previousModifier.map(Algos.encode)}") +// requestDownload(modifierTypeId, Seq(modifiersId), history, isBlockChainSynced, isMining) +// +// case PeersForSyncInfo(peers) => sendSync(history.syncInfo, peers) +// +// case FullBlockChainIsSynced => context.become(basicMessageHandler(history, isBlockChainSynced = true, isMining, checkModScheduler)) +// +// case StartMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = true, checkModScheduler)) +// +// case DisableMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = false, checkModScheduler)) +// +// case UpdatedHistory(historyReader) => context.become(basicMessageHandler(historyReader, isBlockChainSynced, isMining, checkModScheduler)) +// +// case StopTransactionsValidation => canProcessTransactions = false +// +// case StartTransactionsValidation => canProcessTransactions = true +// +// case message => logger.debug(s"Got strange message $message(${message.getClass}) on DeliveryManager from $sender") +// } +// +// /** +// * This function check if modifier has received or not. +// * If modifier has transaction type id, it won't be re-asked. +// * If we still have no this modifier and number of attempts have no expired, we will re-ask it. +// * If we still have no this modifier and number of attempts have expired, we will remove it from expected modifiers collection. +// * Otherwise - do nothing. +// * +// * @param peer - peer, from whom we are expecting modifier +// * @param modifierTypeId - type of expected modifier +// * @param modifierId - expected modifier id +// */ +// def checkDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId): Unit = { +// val expectedModifiersByPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = +// expectedModifiers.getOrElse(peer.socketAddress, Map.empty) +// if (modifierTypeId == Transaction.modifierTypeId) +// expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, toKey(modifierId), peer.socketAddress) +// else expectedModifiersByPeer.find { case (id, (_, _)) => id == toKey(modifierId) } match { +// case Some((_, (_, attempts))) if attempts <= settings.network.maxDeliveryChecks => +// logger.debug(s"Modifier ${Algos.encode(modifierId)} needed to be requested from $peer!") +// reRequestModifier(peer, modifierTypeId, modifierId, expectedModifiersByPeer) +// case Some((modId, (_, _))) => +// logger.debug(s"Maximum number of attempts has expired. Remove modifier ${Algos.encode(modifierId)} from $peer.") +// expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, modId, peer.socketAddress) +// case _ => +// logger.debug(s"This modifiers ${Algos.encode(modifierId)} is not contained in expectedModifiers collection from $peer.") +// } +// } +// +// /** +// * If node is not synced, send sync info to random peer, otherwise to all known peers. +// * +// * @param syncInfo - sync info +// */ +// def sendSync(syncInfo: SyncInfo, peers: Seq[ConnectedPeer]): Unit = peers.foreach { peer => +// logger.info(s"Sending to $peer sync info message.") +// peer.handlerRef ! SyncInfoNetworkMessage(syncInfo) +// } +// +// /** +// * Send request to 'peer' with modifiers ids of type 'modifierTypeId'. +// * We can do this activity only if 'peer' status != Younger. +// * If current chain isn't synced and mining is off, we can't request transactions, otherwise can. +// * +// * We should filter our requesting modifiers to avoid request repeated modifiers. +// * +// * @param history - current history reader +// * @param peer - peer, whom message will be send +// * @param mTypeId - modifier type id +// * @param modifierIds - modifiers ids +// * @param isBlockChainSynced - current block chain status +// * @param isMining - current mining status +// */ +// +// def requestModifies(history: History, +// peer: ConnectedPeer, +// mTypeId: ModifierTypeId, +// modifierIds: Seq[ModifierId], +// isBlockChainSynced: Boolean, +// isMining: Boolean): Unit = { +// val firstCondition: Boolean = mTypeId == Transaction.modifierTypeId && isBlockChainSynced && isMining +// val secondCondition: Boolean = mTypeId != Transaction.modifierTypeId +// val thirdCondition: Boolean = +// if (!isBlockChainSynced) peersCollection.get(peer.socketAddress).exists(p => p._2 != Younger) +// else peersCollection.contains(peer.socketAddress) +// if (mTypeId != Transaction.modifierTypeId) +// logger.debug(s"Got requestModifier for modifiers of type: $mTypeId to $peer with modifiers ${modifierIds.size}." + +// s" Try to check conditions: $firstCondition -> $secondCondition -> $thirdCondition.") +// if ((firstCondition || secondCondition) && thirdCondition) { +// val requestedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers +// .getOrElse(peer.socketAddress, Map.empty) +// +// val notYetRequested: Seq[ModifierId] = modifierIds +// .filter(id => +// !history.isModifierDefined(id) && +// !requestedModifiersFromPeer.contains(toKey(id)) && +// !receivedModifiers.contains(toKey(id)) +// ) +// +// if (notYetRequested.nonEmpty) { +// if (mTypeId != Transaction.modifierTypeId) +// logger.debug(s"Send request to ${peer.socketAddress} for ${notYetRequested.size} modifiers of type $mTypeId ") +// peer.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> notYetRequested) +// priorityCalculator = priorityCalculator.incrementRequestForNModifiers(peer.socketAddress, notYetRequested.size) +// if (mTypeId != Transaction.modifierTypeId) { +// val requestedModIds: Map[ModifierIdAsKey, (Cancellable, Int)] = +// notYetRequested.foldLeft(requestedModifiersFromPeer) { case (rYet, id) => +// rYet.updated(toKey(id), +// context.system +// .scheduler.scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, id)) -> 1) +// } +// expectedModifiers = expectedModifiers.updated(peer.socketAddress, requestedModIds) +// } else expectedTransactions = expectedTransactions ++ modifierIds.map(toKey) +// } +// } +// } +// +// /** +// * Re-ask 'modifierId' from 'peer' if needed. We will do this only if we are expecting these modifier from 'peer' +// * and if number of attempts doesn't expired yet. +// * This activity will update timer on re-asked modifier. +// * +// * @param peer - peer, whom message will be send +// * @param mTypeId - modifier type id +// * @param modId - re-asked modifier id +// */ +// def reRequestModifier(peer: ConnectedPeer, +// mTypeId: ModifierTypeId, +// modId: ModifierId, +// peerRequests: Map[ModifierIdAsKey, (Cancellable, Int)]): Unit = +// peerRequests.get(toKey(modId)) match { +// case Some((_, attempts)) => peersCollection.find { case (innerAddr, (_, cResult, _)) => +// innerAddr == peer.socketAddress && cResult != Younger +// } match { +// case Some((_, (cP, _, _))) => +// cP.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> Seq(modId)) +// logger.debug(s"Re-asked ${peer.socketAddress} and handler: ${peer.handlerRef} for modifier of type: " + +// s"$mTypeId with id: ${Algos.encode(modId)}. Attempts: $attempts") +// priorityCalculator = priorityCalculator.incrementRequest(peer.socketAddress) +// expectedModifiers = expectedModifiers.updated(peer.socketAddress, peerRequests.updated( +// toKey(modId), +// context.system.scheduler +// .scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, modId)) -> (attempts + 1) +// )) +// case None => +// expectedModifiers = clearExpectedModifiersCollection(peerRequests, toKey(modId), peer.socketAddress) +// logger.debug(s"Tried to re-ask modifier ${Algos.encode(modId)}, but this id not needed from this peer") +// } +// case _ => logger.debug(s"There is no such modifier ${Algos.encode(modId)} in expected collection.") +// } +// +// /** +// * Check 'expectedModifiers' for awaiting modifier with id 'mId' from 'peer' +// * +// * @param mId - id of checkable modifier +// * @param peer - peer from which we possibly expecting modifier +// * @return 'true' if we are expecting this modifier from this peer otherwise 'false' +// */ +// def isExpecting(mId: ModifierId, modifierTypeId: ModifierTypeId, peer: ConnectedPeer): Boolean = { +// if (modifierTypeId != Transaction.modifierTypeId) { +// val result: Boolean = expectedModifiers.getOrElse(peer.socketAddress, Map.empty).contains(toKey(mId)) +// logger.debug(s"isExpecting -->> modId ${Algos.encode(mId)} --> $result") +// result +// } else expectedTransactions.contains(toKey(mId)) +// } +// +// /** +// * Clear the 'receivedSpamModifiers' collection +// * +// * @param mIds - sequence of modifiers ids which will be deleted from spam collection +// */ +// def deleteSpam(mIds: Seq[ModifierId]): Unit = for (id <- mIds) receivedSpamModifiers -= toKey(id) +// +// /** +// * Check receivedSpamModifiers for contains received modifier +// * +// * @param mId - checkable modifier +// * @return 'true' if received modifier is in spam collection otherwise 'false' +// */ +// def isSpam(mId: ModifierId): Boolean = receivedSpamModifiers.contains(toKey(mId)) +// +// /** +// * Send inv data to the 'peer'. +// * +// * @param peer - peer whom will send a message +// * @param status - current peer's status +// * @param dataForInvMessage - data for inv message +// */ +// def sendInvData(peer: ConnectedPeer, +// status: HistoryComparisonResult, +// dataForInvMessage: Option[Seq[(ModifierTypeId, ModifierId)]]): Unit = dataForInvMessage match { +// case Some(data) => +// data.groupBy(_._1).mapValues(_.map(_._2)).foreach { +// case (mTid, mods) if mods.size <= settings.network.maxInvObjects => +// logger.debug(s"Send to peer $peer inv msg with mods: ${mods.map(Algos.encode).mkString(",")}") +// peer.handlerRef ! InvNetworkMessage(mTid -> mods) +// case (mTid, mods) => +// val modifiers: Seq[ModifierId] = mods.take(settings.network.maxInvObjects) +// logger.debug(s"Send to peer $peer dropped inv msg with mods: ${modifiers.map(Algos.encode).mkString(",")}") +// peer.handlerRef ! InvNetworkMessage(mTid -> modifiers) +// } +// case None => logger.info(s"dataForInvMessage is empty for: $peer. Peer's status is: $status.") +// } +// +// /** +// * If node is not synced, `requestDownload` sends request for the one peer which will be find by 2 criteria: +// * 1) HistoryComparisonResult != Younger. +// * 2) Choose random peer with non bad priority. +// * Otherwise this function sends requests for all known peers selected by 1-st criterion as above. +// * +// * If there are no any peers, request won't be sent. +// * +// * @param modifierTypeId - modifier type id +// * @param modifierIds - modifier id +// * @param history - current history state +// * @param isBlockChainSynced - current block chain status +// * @param isMining - current mining status +// */ +// def requestDownload(modifierTypeId: ModifierTypeId, +// modifierIds: Seq[ModifierId], +// history: History, +// isBlockChainSynced: Boolean, +// isMining: Boolean): Unit = +// if (!isBlockChainSynced) { +// logger.debug(s"requestDownload -> !isBlockChainSynced = true") +// val (withBadNodesMap, withoutBadNodesMap) = peersCollection.filter(p => p._2._2 != Younger).partition { +// case (_, (_, _, priority)) => priority == BadNode +// } +// logger.debug(s"withBadNodesMap -> ${withBadNodesMap.keys.mkString(",")}") +// logger.debug(s"withoutBadNodesMap -> ${withoutBadNodesMap.keys.mkString(",")}") +// val withBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = +// withBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq +// val withoutBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = +// withoutBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq +// val resultedPeerCollection = +// if (withBadNodes.nonEmpty) withoutBadNodes :+ Random.shuffle(withBadNodes).head +// else withoutBadNodes +// logger.debug(s"resultedPeerCollection -> $resultedPeerCollection") +// logger.debug(s"Block chain is not synced. acceptedPeers: $resultedPeerCollection") +// if (resultedPeerCollection.nonEmpty) { +// val shuffle: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = Random.shuffle(resultedPeerCollection) +// val cP = shuffle.last._1 +// influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.debug(s"requestModifies for peer ${cP.socketAddress} for mods: ${modifierIds.map(Algos.encode).mkString(",")}") +// requestModifies(history, cP, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// } else logger.info(s"BlockChain is not synced. There is no nodes, which we can connect with.") +// } +// else peersCollection.filter(p => p._2._2 != Younger) match { +// case coll: Map[_, _] if coll.nonEmpty => +// influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) +// coll.foreach { case (_, (cp, _, _)) => +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.info(s"Sent download request to the ${cp.socketAddress} to modifiers of type: $modifierTypeId.") +// requestModifies(history, cp, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// } +// case _ => logger.info(s"BlockChain is synced. There is no nodes, which we can connect with.") +// } +// +// /** +// * Handle received modifier. We will process received modifier only if we are expecting this on. +// * +// * @param mTid - modifier type id +// * @param mId - modifier id +// * @param peer - peer who sent modifier +// * @param isBlockChainSynced - current chain status +// */ +// def receive(mTid: ModifierTypeId, +// mId: ModifierId, +// peer: ConnectedPeer, +// isBlockChainSynced: Boolean): Unit = +// if (isExpecting(mId, mTid, peer)) { +// if (mTid != Transaction.modifierTypeId) { +// logger.debug(s"Got new modifier with type $mTid from: ${peer.socketAddress}. with id ${Algos.encode(mId)}") +// } +// priorityCalculator = priorityCalculator.incrementReceive(peer.socketAddress) +// val peerExpectedModifiers: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers +// .getOrElse(peer.socketAddress, Map.empty) +// peerExpectedModifiers.get(toKey(mId)).foreach(_._1.cancel()) +// if (mTid != Transaction.modifierTypeId) receivedModifiers += toKey(mId) +// if (mTid != Transaction.modifierTypeId) expectedModifiers = clearExpectedModifiersCollection(peerExpectedModifiers, toKey(mId), peer.socketAddress) +// else expectedTransactions = expectedTransactions - toKey(mId) +// } else { +// receivedSpamModifiers = receivedSpamModifiers - toKey(mId) + (toKey(mId) -> peer) +// priorityCalculator = priorityCalculator.decrementRequest(peer.socketAddress) +// } +// +// /** +// * Transform modifier id to WrappedArray.ofBytes +// * +// * @param id - modifier id which will be transform to WrappedArray of bytes. +// * @return transformed modifier id +// */ +// def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) +// +// /** +// * This function gets collection of current expected modifiers from 'peer' and modifier, which +// * will be removed from received collection as a parameters. +// * If expected modifiers collection will contain other modifiers even after removing, +// * this function will return collection of expectedModifiers with updated 'peer' expected collection +// * otherwise it will return expectedModifiers collection without 'peer'. +// * +// * @param expectedModifiersFromPeer - collection of expected modifiers from 'peer' +// * @param modifierId - modifier id, which will be removed from 'expectedModifiersFromPeer' +// * @param peer - 'peer' from which expected modifiers collection we remove received modifier +// * @return - expectedModifiers collection without 'peer' or expectedModifiers with updated 'peer' expected collection +// */ +// def clearExpectedModifiersCollection(expectedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)], +// modifierId: ModifierIdAsKey, +// peer: InetSocketAddress): Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = { +// val collectionWithoutModId: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiersFromPeer - modifierId +// collectionWithoutModId match { +// case coll: Map[_, _] if coll.nonEmpty => expectedModifiers.updated(peer, coll) +// case _ => expectedModifiers - peer +// } +// } +//} +// object DeliveryManager { final case class CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) @@ -515,15 +516,15 @@ object DeliveryManager { final case class CheckModifiersWithQueueSize(size: Int) extends AnyVal - def props(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - networkControllerRef: ActorRef, - memoryPoolRef: ActorRef, - nodeViewSync: ActorRef, - downloadedModifiersValidator: ActorRef, - settings: EncryAppSettings): Props = - Props(new DeliveryManager(influxRef, nodeViewHolderRef, networkControllerRef, memoryPoolRef, nodeViewSync, - downloadedModifiersValidator, settings)) +// def props(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// networkControllerRef: ActorRef, +// memoryPoolRef: ActorRef, +// nodeViewSync: ActorRef, +// downloadedModifiersValidator: ActorRef, +// settings: EncryAppSettings): Props = +// Props(new DeliveryManager(influxRef, nodeViewHolderRef, networkControllerRef, memoryPoolRef, nodeViewSync, +// downloadedModifiersValidator, settings)) class DeliveryManagerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( @@ -542,11 +543,11 @@ object DeliveryManager { case InvalidModifier(_) => 2 - case DataFromPeer(msg: ModifiersNetworkMessage, _) => - msg match { - case ModifiersNetworkMessage((typeId, _)) if typeId != Transaction.modifierTypeId => 1 - case _ => 3 - } +// case DataFromPeer(msg: ModifiersNetworkMessage, _) => +// msg match { +// case ModifiersNetworkMessage((typeId, _)) if typeId != Transaction.modifierTypeId => 1 +// case _ => 3 +// } case PoisonPill => 4 diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 70999f12b6..3c4651e727 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -20,45 +20,45 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.util.{Failure, Success, Try} -class DownloadedModifiersValidator(modifierIdSize: Int, - nodeViewHolder: ActorRef, - peersKeeper: ActorRef, - nodeViewSync: ActorRef, - memoryPoolRef: ActorRef, - influxRef: Option[ActorRef], - settings: EncryAppSettings) - extends Actor - with StrictLogging { - - override def receive: Receive = { - case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) - case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") - } - - def workingCycle(history: History): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - typeId match { - case Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { - case Success(tx) if tx.semanticValidity.isSuccess => memoryPoolRef ! NewTransaction(tx) - case Success(tx) => - logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") - context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) - nodeViewSync ! InvalidModifier(id) - case Failure(ex) => - context.parent ! BanPeer(remote, CorruptedSerializedBytes) - nodeViewSync ! InvalidModifier(id) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - } - } - } - case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) - case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") - } - -} +//class DownloadedModifiersValidator(modifierIdSize: Int, +// nodeViewHolder: ActorRef, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef, +// memoryPoolRef: ActorRef, +// influxRef: Option[ActorRef], +// settings: EncryAppSettings) +// extends Actor +// with StrictLogging { +// +// override def receive: Receive = { +// case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) +// case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") +// } +// +// def workingCycle(history: History): Receive = { +// case ModifiersForValidating(remote, typeId, filteredModifiers) => +// typeId match { +// case Transaction.modifierTypeId => +// filteredModifiers.foreach { +// case (id, bytes) => +// Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { +// case Success(tx) if tx.semanticValidity.isSuccess => memoryPoolRef ! NewTransaction(tx) +// case Success(tx) => +// logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") +// context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) +// nodeViewSync ! InvalidModifier(id) +// case Failure(ex) => +// context.parent ! BanPeer(remote, CorruptedSerializedBytes) +// nodeViewSync ! InvalidModifier(id) +// logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") +// } +// } +// } +// case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) +// case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") +// } +// +//} object DownloadedModifiersValidator { @@ -68,22 +68,22 @@ object DownloadedModifiersValidator { final case class InvalidModifier(ids: ModifierId) extends AnyVal - def props(modifierIdSize: Int, - nodeViewHolder: ActorRef, - peersKeeper: ActorRef, - nodeViewSync: ActorRef, - memoryPoolRef: ActorRef, - influxRef: Option[ActorRef], - settings: EncryAppSettings): Props = - Props( - new DownloadedModifiersValidator(modifierIdSize, - nodeViewHolder, - peersKeeper, - nodeViewSync, - memoryPoolRef, - influxRef, - settings) - ) +// def props(modifierIdSize: Int, +// nodeViewHolder: ActorRef, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef, +// memoryPoolRef: ActorRef, +// influxRef: Option[ActorRef], +// settings: EncryAppSettings): Props = +// Props( +// new DownloadedModifiersValidator(modifierIdSize, +// nodeViewHolder, +// peersKeeper, +// nodeViewSync, +// memoryPoolRef, +// influxRef, +// settings) +// ) class DownloadedModifiersValidatorPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox(PriorityGenerator { diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 4df0ef55a9..952e95fcb9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -5,16 +5,21 @@ import java.net.InetSocketAddress import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout +import com.typesafe.scalalogging.StrictLogging +import encry.network.DM.{IsRequested, RequestSent} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ import scala.util.Try -case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { +case class MessageBuilder(msg: MessageToNetwork, + peersKeeper: ActorRef, + deliveryManager: ActorRef) extends Actor with StrictLogging { import context.dispatcher @@ -23,8 +28,15 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends override def receive: Receive = { case RequestFromLocal(peer, modTypeId, modsIds) => Try { - (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + modsIds.foreach { modId => + for { + isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] + } yield if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } } } case SendSyncInfo(syncInfo) => @@ -49,5 +61,7 @@ object MessageBuilder { case object GetPeers case class GetPeerInfo(peerIp: InetSocketAddress) - def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) + def props(msg: MessageToNetwork, + peersKeeper: ActorRef, + deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 0213394830..36d63e9798 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -20,121 +20,121 @@ import scala.concurrent.duration._ import scala.language.{existentials, postfixOps} import scala.util.Try -class NetworkController(networkSettings: NetworkSettings, - peersKeeper: ActorRef, - nodeViewSync: ActorRef) extends Actor with StrictLogging { - - import context.dispatcher - import context.system - - override def preStart(): Unit = logger.info(s"Network controller started") - - var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty - val externalSocketAddress: Option[InetSocketAddress] = networkSettings.declaredAddress - logger.info(s"Declared address is: $externalSocketAddress.") - - if (!networkSettings.localOnly.getOrElse(false)) networkSettings.declaredAddress.foreach(myAddress => - Try(NetworkInterface.getNetworkInterfaces.asScala.exists(interface => - interface.getInterfaceAddresses.asScala.exists(interfaceAddress => - InetAddress.getAllByName(new URI("http://" + myAddress).getHost).contains(interfaceAddress.getAddress) - ))).recover { case t: Throwable => logger.error(s"Declared address validation failed: $t") } - ) - - IO(Tcp) ! Bind(self, networkSettings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - - override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy( - maxNrOfRetries = 5, - withinTimeRange = 60 seconds) { - case _ => Restart - } - - override def receive: Receive = bindingLogic - .orElse(businessLogic) - .orElse(peersLogic) - .orElse { - case RegisterMessagesHandler(types, handler) => - logger.info(s"Registering handlers for ${types.mkString(",")}.") - val ids = types.map(_._1) - messagesHandlers += (ids -> handler) - case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") - case msg => logger.warn(s"NetworkController: got something strange $msg.") - } - - def bindingLogic: Receive = { - case Bound(address) => - logger.info(s"Successfully bound to the port ${address.getPort}.") - context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) - case CommandFailed(add: Bind) => - logger.info(s"Node can't be bind to the address: ${add.localAddress}.") - context.stop(self) - } - - def businessLogic: Receive = { - case MessageFromNetwork(message, Some(remote)) if message.isValid(networkSettings.syncPacketLength) => - logger.debug(s"Got ${message.messageName} on the NetworkController.") - findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) - case MessageFromNetwork(message, Some(remote)) => - peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) - logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") - } - - def peersLogic: Receive = { - case PeerForConnection(peer) => - logger.info(s"Network controller got new peer for connection: $peer. Trying to set connection with remote...") - IO(Tcp) ! Connect( - peer, - None, - KeepAlive(true) :: Nil, - Some(networkSettings.connectionTimeout), - pullMode = true - ) - - case Connected(remote, localAddress) => - logger.info(s"Network controller got 'Connected' message from: $remote. " + - s"Trying to set stable connection with remote... " + - s"Local TCP endpoint is: $localAddress.") - peersKeeper ! NewConnection(remote, sender()) - - case ConnectionVerified(remote, remoteConnection, connectionType) => - logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") - val peerConnectionHandler: ActorRef = context.actorOf( - PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, networkSettings) - .withDispatcher("network-dispatcher") - ) - peerConnectionHandler ! StartInteraction - - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - - case ConnectionStopped(peer) => - logger.info(s"Network controller got signal about breaking connection with: $peer. " + - s"Sending to peerKeeper actual information.") - peersKeeper ! ConnectionStopped(peer) - nodeViewSync ! ConnectionStopped(peer) - - case CommandFailed(connect: Connect) => - logger.info(s"Failed to connect to: ${connect.remoteAddress}.") - peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) - } - - private def findHandler(message: NetworkMessage, - messageId: Byte, - remote: ConnectedPeer, - mH: Map[Seq[Byte], ActorRef]): Unit = - mH.find(_._1.contains(messageId)).map(_._2) match { - case Some(handler) => - handler ! DataFromPeer(message, remote) - logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") - case None => logger.info("No handlers found for message: " + message.messageName) - } -} +//class NetworkController(networkSettings: NetworkSettings, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef) extends Actor with StrictLogging { +// +// import context.dispatcher +// import context.system +// +// override def preStart(): Unit = logger.info(s"Network controller started") +// +// var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty +// val externalSocketAddress: Option[InetSocketAddress] = networkSettings.declaredAddress +// logger.info(s"Declared address is: $externalSocketAddress.") +// +// if (!networkSettings.localOnly.getOrElse(false)) networkSettings.declaredAddress.foreach(myAddress => +// Try(NetworkInterface.getNetworkInterfaces.asScala.exists(interface => +// interface.getInterfaceAddresses.asScala.exists(interfaceAddress => +// InetAddress.getAllByName(new URI("http://" + myAddress).getHost).contains(interfaceAddress.getAddress) +// ))).recover { case t: Throwable => logger.error(s"Declared address validation failed: $t") } +// ) +// +// IO(Tcp) ! Bind(self, networkSettings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) +// +// override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy( +// maxNrOfRetries = 5, +// withinTimeRange = 60 seconds) { +// case _ => Restart +// } +// +// override def receive: Receive = bindingLogic +// .orElse(businessLogic) +// .orElse(peersLogic) +// .orElse { +// case RegisterMessagesHandler(types, handler) => +// logger.info(s"Registering handlers for ${types.mkString(",")}.") +// val ids = types.map(_._1) +// messagesHandlers += (ids -> handler) +// case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") +// case msg => logger.warn(s"NetworkController: got something strange $msg.") +// } +// +// def bindingLogic: Receive = { +// case Bound(address) => +// logger.info(s"Successfully bound to the port ${address.getPort}.") +// context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) +// case CommandFailed(add: Bind) => +// logger.info(s"Node can't be bind to the address: ${add.localAddress}.") +// context.stop(self) +// } +// +// def businessLogic: Receive = { +// case MessageFromNetwork(message, Some(remote)) if message.isValid(networkSettings.syncPacketLength) => +// logger.debug(s"Got ${message.messageName} on the NetworkController.") +// findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) +// case MessageFromNetwork(message, Some(remote)) => +// peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) +// logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") +// } +// +// def peersLogic: Receive = { +// case PeerForConnection(peer) => +// logger.info(s"Network controller got new peer for connection: $peer. Trying to set connection with remote...") +// IO(Tcp) ! Connect( +// peer, +// None, +// KeepAlive(true) :: Nil, +// Some(networkSettings.connectionTimeout), +// pullMode = true +// ) +// +// case Connected(remote, localAddress) => +// logger.info(s"Network controller got 'Connected' message from: $remote. " + +// s"Trying to set stable connection with remote... " + +// s"Local TCP endpoint is: $localAddress.") +// peersKeeper ! NewConnection(remote, sender()) +// +// case ConnectionVerified(remote, remoteConnection, connectionType) => +// logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") +// val peerConnectionHandler: ActorRef = context.actorOf( +// PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, networkSettings) +// .withDispatcher("network-dispatcher") +// ) +// peerConnectionHandler ! StartInteraction +// +// case HandshakedDone(remote) => +// logger.info(s"Network controller got approvement from peer handler about successful handshake. " + +// s"Sending to peerKeeper connected peer.") +// peersKeeper ! HandshakedDone(remote) +// +// case ConnectionStopped(peer) => +// logger.info(s"Network controller got signal about breaking connection with: $peer. " + +// s"Sending to peerKeeper actual information.") +// peersKeeper ! ConnectionStopped(peer) +// nodeViewSync ! ConnectionStopped(peer) +// +// case CommandFailed(connect: Connect) => +// logger.info(s"Failed to connect to: ${connect.remoteAddress}.") +// peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) +// } +// +// private def findHandler(message: NetworkMessage, +// messageId: Byte, +// remote: ConnectedPeer, +// mH: Map[Seq[Byte], ActorRef]): Unit = +// mH.find(_._1.contains(messageId)).map(_._2) match { +// case Some(handler) => +// handler ! DataFromPeer(message, remote) +// logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") +// case None => logger.info("No handlers found for message: " + message.messageName) +// } +//} object NetworkController { - def props(networkSettings: NetworkSettings, peersKeeper: ActorRef, nodeViewSync: ActorRef): Props = - Props(new NetworkController(networkSettings, peersKeeper, nodeViewSync)) +// def props(networkSettings: NetworkSettings, peersKeeper: ActorRef, nodeViewSync: ActorRef): Props = +// Props(new NetworkController(networkSettings, peersKeeper, nodeViewSync)) object ReceivableMessages { diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index f815b1db88..2cd0ed37cd 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -11,18 +11,21 @@ import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} -import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} +import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.concurrent.duration._ class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { import context.system + import context.dispatcher var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty var handlerForMods: ActorRef = ActorRef.noSender @@ -30,10 +33,10 @@ class NetworkRouter(settings: NetworkSettings, IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") - val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") + val deliveryManager = context.system.actorOf(DM.props(settings), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress - override def receive: Receive = bindingLogic orElse businessLogic orElse { + override def receive: Receive = bindingLogic orElse businessLogic orElse peersLogic orElse { case RegisterMessagesHandler(types, handler) => logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) @@ -46,6 +49,7 @@ class NetworkRouter(settings: NetworkSettings, def bindingLogic: Receive = { case Bound(address) => logger.info(s"Successfully bound to the port ${address.getPort}.") + context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) case CommandFailed(add: Bind) => logger.info(s"Node can't be bind to the address: ${add.localAddress}.") context.stop(self) @@ -59,7 +63,8 @@ class NetworkRouter(settings: NetworkSettings, peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: ModifierFromNetwork => handlerForMods ! msg - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") + case msg: OtherNodeSyncingStatus => peersKeeper ! msg + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), "peersKeeper") } def peersLogic: Receive = { @@ -113,5 +118,6 @@ object NetworkRouter { case object RegisterForModsHandling - def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) + def props(settings: NetworkSettings, + blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 3f11d22ae2..8dc068636f 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -37,181 +37,181 @@ import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} import scala.util.Try -class NodeViewSynchronizer(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - settings: EncryAppSettings, - memoryPoolRef: ActorRef, - dataHolder: ActorRef) extends Actor with StrictLogging { - - val peersKeeper: ActorRef = context.system.actorOf(PeersKeeper.props(settings, self, dataHolder) - .withDispatcher("peers-keeper-dispatcher"), "PeersKeeper") - - - val networkController: ActorRef = context.system.actorOf(NetworkController.props(settings.network, peersKeeper, self) - .withDispatcher("network-dispatcher"), "NetworkController") - - val snapshotHolder: ActorRef = context.system.actorOf(SnapshotHolder.props(settings, networkController, nodeViewHolderRef, self) - .withDispatcher("snapshot-holder-dispatcher"), "snapshotHolder") - - networkController ! RegisterMessagesHandler(Seq( - InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", - RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", - SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" - ), self) - - implicit val timeout: Timeout = Timeout(5.seconds) - - var historyReaderOpt: Option[History] = None - var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty - var chainSynced: Boolean = false - - var canProcessTransactions: Boolean = true - - val downloadedModifiersValidator: ActorRef = context.system - .actorOf(DownloadedModifiersValidator.props(settings.constants.ModifierIdSize, nodeViewHolderRef, - peersKeeper, self, memoryPoolRef, influxRef, settings) - .withDispatcher("Downloaded-Modifiers-Validator-dispatcher"), "DownloadedModifiersValidator") - - val deliveryManager: ActorRef = context.actorOf( - DeliveryManager.props(influxRef, nodeViewHolderRef, networkController, memoryPoolRef, self, - downloadedModifiersValidator, settings) - .withDispatcher("delivery-manager-dispatcher"), "DeliveryManager") - - override def preStart(): Unit = { - context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) - context.system.eventStream.subscribe(self, classOf[ClIMiner]) - context.system.eventStream.subscribe(self, classOf[CLIPeer]) - nodeViewHolderRef ! GetNodeViewChanges(history = true, state = false, vault = false) - } - - override def receive: Receive = awaitingHistoryCycle - - def awaitingHistoryCycle: Receive = { - case msg@ChangedHistory(reader: History) => - logger.info(s"get history: $reader from $sender") - deliveryManager ! UpdatedHistory(reader) - snapshotHolder ! msg - downloadedModifiersValidator ! UpdatedHistory(reader) - context.become(workingCycle(reader)) - case msg@RegisterMessagesHandler(_, _) => networkController ! msg - case msg => logger.info(s"Nvsh got strange message: $msg during history awaiting.") - } - - def workingCycle(history: History): Receive = { - case msg@InvalidModifier(_) => deliveryManager ! msg - case msg@RegisterMessagesHandler(_, _) => networkController ! msg - case SemanticallySuccessfulModifier(mod) => mod match { - case block: Block if chainSynced => - broadcastModifierInv(block.header) - broadcastModifierInv(block.payload) - modifiersRequestCache = Map( - Algos.encode(block.id) -> toProto(block.header), - Algos.encode(block.payload.id) -> toProto(block.payload) - ) - case tx: Transaction => broadcastModifierInv(tx) - case _ => //Do nothing - } - case DataFromPeer(message, remote) => message match { - - case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => - val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds - .flatMap(id => modifiersRequestCache - .get(Algos.encode(id)) - .map(id -> _)) - .toMap - if (modifiersFromCache.nonEmpty) remote.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersFromCache) - val unrequestedModifiers: Seq[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains) - - if (unrequestedModifiers.nonEmpty) typeId match { - case Transaction.modifierTypeId => - memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) - } - - case RequestModifiersNetworkMessage(requestedIds) => - logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") - - case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId && chainSynced && canProcessTransactions => - memoryPoolRef ! CompareViews(remote, invData._1, invData._2) - case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => - logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + - s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - - case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") - } - case msg@RequestPeersForFirstSyncInfo => - logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + - s" peers for first sync info message. Resending $msg to peers keeper.") - peersKeeper ! msg - case msg@UpdatedPeersCollection(_) => deliveryManager ! msg - case msg@PeersForSyncInfo(_) => - logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") - deliveryManager ! msg - case msg@TreeChunks(l, b) => snapshotHolder ! msg - case msg@ConnectionStopped(_) => deliveryManager ! msg - case msg@StartMining => deliveryManager ! msg - case msg@DisableMining => deliveryManager ! msg - case msg@BanPeer(_, _) => peersKeeper ! msg - case msg@AccumulatedPeersStatistic(_) => peersKeeper ! msg - case msg@SendLocalSyncInfo => peersKeeper ! msg - case msg@RemovePeerFromBlackList(_) => peersKeeper ! msg - case msg@RequiredManifestHeightAndId(_, _) => snapshotHolder ! msg - case msg@SendToNetwork(_, _) => - logger.info(s"NVSH got SendToNetwork") - peersKeeper ! msg - case msg@HeaderChainIsSynced => - snapshotHolder ! msg - case msg@UpdateSnapshot(_, _) => snapshotHolder ! msg - case msg@FastSyncDone => snapshotHolder ! FastSyncDone - case ChangedHistory(reader: History@unchecked) if reader.isInstanceOf[History] => - deliveryManager ! UpdatedHistory(reader) - downloadedModifiersValidator ! UpdatedHistory(reader) - context.become(workingCycle(reader)) - case RequestedModifiersForRemote(remote, txs) => sendResponse( - remote, Transaction.modifierTypeId, txs.map(tx => tx.id -> TransactionProtoSerializer.toProto(tx).toByteArray) - ) - case SuccessfulTransaction(tx) => broadcastModifierInv(tx) - case SemanticallyFailedModification(_, _) => - case SyntacticallyFailedModification(_, _) => - case msg@PeerFromCli(peer) => peersKeeper ! msg - case FullBlockChainIsSynced => - chainSynced = true - deliveryManager ! FullBlockChainIsSynced - peersKeeper ! FullBlockChainIsSynced - if (!settings.snapshotSettings.enableFastSynchronization) snapshotHolder ! FullBlockChainIsSynced - case StopTransactionsValidation => - deliveryManager ! StopTransactionsValidation - canProcessTransactions = false - case StartTransactionsValidation => - deliveryManager ! StartTransactionsValidation - canProcessTransactions = true - case a: Any => logger.error(s"Strange input(sender: ${sender()}): ${a.getClass}\n" + a) - } - - def sendResponse(peer: ConnectedPeer, typeId: ModifierTypeId, modifiersBytes: Seq[(ModifierId, Array[Byte])]): Unit = - if (modifiersBytes.nonEmpty) { - if (typeId != Transaction.modifierTypeId) - logger.debug(s"Sent modifiers to $peer size is: ${modifiersBytes.length}") - typeId match { - case Header.modifierTypeId => - logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for HEADERS with ${modifiersBytes.size} headers." + - s" \n Headers are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - case Payload.modifierTypeId => - logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for PAYLOADS with ${modifiersBytes.size} payloads." + - s" Mods length: ${modifiersBytes.map(_._2.length).mkString(",")}" + - s" \n Payloads are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - case Transaction.modifierTypeId => - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - } - } - - def broadcastModifierInv(m: NodeViewModifier): Unit = - if (chainSynced) { - logger.debug(s"NVSH is synced. Going to broadcast inv for: ${m.encodedId}") - peersKeeper ! SendToNetwork(InvNetworkMessage(m.modifierTypeId -> Seq(m.id)), Broadcast) - } -} +//class NodeViewSynchronizer(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// settings: EncryAppSettings, +// memoryPoolRef: ActorRef, +// dataHolder: ActorRef) extends Actor with StrictLogging { +// +// val peersKeeper: ActorRef = context.system.actorOf(PeersKeeper.props(settings, self, dataHolder) +// .withDispatcher("peers-keeper-dispatcher"), "PeersKeeper") +// +// +// val networkController: ActorRef = context.system.actorOf(NetworkController.props(settings.network, peersKeeper, self) +// .withDispatcher("network-dispatcher"), "NetworkController") +// +// val snapshotHolder: ActorRef = context.system.actorOf(SnapshotHolder.props(settings, networkController, nodeViewHolderRef, self) +// .withDispatcher("snapshot-holder-dispatcher"), "snapshotHolder") +// +// networkController ! RegisterMessagesHandler(Seq( +// InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", +// RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", +// SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" +// ), self) +// +// implicit val timeout: Timeout = Timeout(5.seconds) +// +// var historyReaderOpt: Option[History] = None +// var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty +// var chainSynced: Boolean = false +// +// var canProcessTransactions: Boolean = true +// +// val downloadedModifiersValidator: ActorRef = context.system +// .actorOf(DownloadedModifiersValidator.props(settings.constants.ModifierIdSize, nodeViewHolderRef, +// peersKeeper, self, memoryPoolRef, influxRef, settings) +// .withDispatcher("Downloaded-Modifiers-Validator-dispatcher"), "DownloadedModifiersValidator") +// +// val deliveryManager: ActorRef = context.actorOf( +// DeliveryManager.props(influxRef, nodeViewHolderRef, networkController, memoryPoolRef, self, +// downloadedModifiersValidator, settings) +// .withDispatcher("delivery-manager-dispatcher"), "DeliveryManager") +// +// override def preStart(): Unit = { +// context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) +// context.system.eventStream.subscribe(self, classOf[ClIMiner]) +// context.system.eventStream.subscribe(self, classOf[CLIPeer]) +// nodeViewHolderRef ! GetNodeViewChanges(history = true, state = false, vault = false) +// } +// +// override def receive: Receive = awaitingHistoryCycle +// +// def awaitingHistoryCycle: Receive = { +// case msg@ChangedHistory(reader: History) => +// logger.info(s"get history: $reader from $sender") +// deliveryManager ! UpdatedHistory(reader) +// snapshotHolder ! msg +// downloadedModifiersValidator ! UpdatedHistory(reader) +// context.become(workingCycle(reader)) +// case msg@RegisterMessagesHandler(_, _) => networkController ! msg +// case msg => logger.info(s"Nvsh got strange message: $msg during history awaiting.") +// } +// +// def workingCycle(history: History): Receive = { +// case msg@InvalidModifier(_) => deliveryManager ! msg +// case msg@RegisterMessagesHandler(_, _) => networkController ! msg +// case SemanticallySuccessfulModifier(mod) => mod match { +// case block: Block if chainSynced => +// broadcastModifierInv(block.header) +// broadcastModifierInv(block.payload) +// modifiersRequestCache = Map( +// Algos.encode(block.id) -> toProto(block.header), +// Algos.encode(block.payload.id) -> toProto(block.payload) +// ) +// case tx: Transaction => broadcastModifierInv(tx) +// case _ => //Do nothing +// } +// case DataFromPeer(message, remote) => message match { +// +// case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => +// val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds +// .flatMap(id => modifiersRequestCache +// .get(Algos.encode(id)) +// .map(id -> _)) +// .toMap +// if (modifiersFromCache.nonEmpty) remote.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersFromCache) +// val unrequestedModifiers: Seq[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains) +// +// if (unrequestedModifiers.nonEmpty) typeId match { +// case Transaction.modifierTypeId => +// memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) +// } +// +// case RequestModifiersNetworkMessage(requestedIds) => +// logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") +// +// case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId && chainSynced && canProcessTransactions => +// memoryPoolRef ! CompareViews(remote, invData._1, invData._2) +// case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => +// logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + +// s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") +// +// case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") +// } +// case msg@RequestPeersForFirstSyncInfo => +// logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + +// s" peers for first sync info message. Resending $msg to peers keeper.") +// peersKeeper ! msg +// case msg@UpdatedPeersCollection(_) => deliveryManager ! msg +// case msg@PeersForSyncInfo(_) => +// logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") +// deliveryManager ! msg +// case msg@TreeChunks(l, b) => snapshotHolder ! msg +// case msg@ConnectionStopped(_) => deliveryManager ! msg +// case msg@StartMining => deliveryManager ! msg +// case msg@DisableMining => deliveryManager ! msg +// case msg@BanPeer(_, _) => peersKeeper ! msg +// case msg@AccumulatedPeersStatistic(_) => peersKeeper ! msg +// case msg@SendLocalSyncInfo => peersKeeper ! msg +// case msg@RemovePeerFromBlackList(_) => peersKeeper ! msg +// case msg@RequiredManifestHeightAndId(_, _) => snapshotHolder ! msg +// case msg@SendToNetwork(_, _) => +// logger.info(s"NVSH got SendToNetwork") +// peersKeeper ! msg +// case msg@HeaderChainIsSynced => +// snapshotHolder ! msg +// case msg@UpdateSnapshot(_, _) => snapshotHolder ! msg +// case msg@FastSyncDone => snapshotHolder ! FastSyncDone +// case ChangedHistory(reader: History@unchecked) if reader.isInstanceOf[History] => +// deliveryManager ! UpdatedHistory(reader) +// downloadedModifiersValidator ! UpdatedHistory(reader) +// context.become(workingCycle(reader)) +// case RequestedModifiersForRemote(remote, txs) => sendResponse( +// remote, Transaction.modifierTypeId, txs.map(tx => tx.id -> TransactionProtoSerializer.toProto(tx).toByteArray) +// ) +// case SuccessfulTransaction(tx) => broadcastModifierInv(tx) +// case SemanticallyFailedModification(_, _) => +// case SyntacticallyFailedModification(_, _) => +// case msg@PeerFromCli(peer) => peersKeeper ! msg +// case FullBlockChainIsSynced => +// chainSynced = true +// deliveryManager ! FullBlockChainIsSynced +// peersKeeper ! FullBlockChainIsSynced +// if (!settings.snapshotSettings.enableFastSynchronization) snapshotHolder ! FullBlockChainIsSynced +// case StopTransactionsValidation => +// deliveryManager ! StopTransactionsValidation +// canProcessTransactions = false +// case StartTransactionsValidation => +// deliveryManager ! StartTransactionsValidation +// canProcessTransactions = true +// case a: Any => logger.error(s"Strange input(sender: ${sender()}): ${a.getClass}\n" + a) +// } +// +// def sendResponse(peer: ConnectedPeer, typeId: ModifierTypeId, modifiersBytes: Seq[(ModifierId, Array[Byte])]): Unit = +// if (modifiersBytes.nonEmpty) { +// if (typeId != Transaction.modifierTypeId) +// logger.debug(s"Sent modifiers to $peer size is: ${modifiersBytes.length}") +// typeId match { +// case Header.modifierTypeId => +// logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for HEADERS with ${modifiersBytes.size} headers." + +// s" \n Headers are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// case Payload.modifierTypeId => +// logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for PAYLOADS with ${modifiersBytes.size} payloads." + +// s" Mods length: ${modifiersBytes.map(_._2.length).mkString(",")}" + +// s" \n Payloads are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// case Transaction.modifierTypeId => +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// } +// } +// +// def broadcastModifierInv(m: NodeViewModifier): Unit = +// if (chainSynced) { +// logger.debug(s"NVSH is synced. Going to broadcast inv for: ${m.encodedId}") +// peersKeeper ! SendToNetwork(InvNetworkMessage(m.modifierTypeId -> Seq(m.id)), Broadcast) +// } +//} object NodeViewSynchronizer { @@ -257,23 +257,23 @@ object NodeViewSynchronizer { } - def props(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - settings: EncryAppSettings, - memoryPoolRef: ActorRef, - dataHolder: ActorRef): Props = - Props(new NodeViewSynchronizer(influxRef, nodeViewHolderRef, settings, memoryPoolRef, dataHolder)) +// def props(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// settings: EncryAppSettings, +// memoryPoolRef: ActorRef, +// dataHolder: ActorRef): Props = +// Props(new NodeViewSynchronizer(influxRef, nodeViewHolderRef, settings, memoryPoolRef, dataHolder)) class NodeViewSynchronizerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case DataFromPeer(msg, _) => msg match { - case SyncInfoNetworkMessage(_) => 1 - case InvNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 1 - case RequestModifiersNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 2 - case _ => 4 - } +// case DataFromPeer(msg, _) => msg match { +// case SyncInfoNetworkMessage(_) => 1 +// case InvNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 1 +// case RequestModifiersNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 2 +// case _ => 4 +// } case SemanticallySuccessfulModifier(mod) => mod match { case _: Transaction => 4 diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index fd67067aed..c8608160d0 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -4,14 +4,15 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} -import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI} +import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} -import scala.util.Try +import scala.util.{Random, Try} class PK(networkSettings: NetworkSettings, blacklistSettings: BlackListSettings) extends Actor with StrictLogging { @@ -33,6 +34,30 @@ class PK(networkSettings: NetworkSettings, .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap override def receive: Receive = banPeersLogic orElse { + case RequestPeerForConnection if connectedPeers.size < networkSettings.maxConnections => + def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r + logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + + s"so peer keeper allows to add one more connection. Current available peers are: " + + s"${peersForConnection.mkString(",")}. Current black list is: ${ + blackList.collect((_, _, _, _) => true, mapReason).mkString(",") + }. Current known peers: ${knownPeers.mkString(",")}.") + logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") + logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") + val peers = peersForConnection + .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) + logger.info(s"peers size: ${peers.size}") + Random.shuffle(peers.toSeq) + .headOption + .foreach { case (peer, _) => + outgoingConnections += peer + logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + + s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + + s"${outgoingConnections.mkString(",")}.") + sender() ! PeerForConnection(peer) + awaitingHandshakeConnections += peer + logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") + } case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + diff --git a/src/main/scala/encry/network/PeerConnectionHandler.scala b/src/main/scala/encry/network/PeerConnectionHandler.scala index 55b9e0c065..b11a23b428 100755 --- a/src/main/scala/encry/network/PeerConnectionHandler.scala +++ b/src/main/scala/encry/network/PeerConnectionHandler.scala @@ -12,7 +12,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp.timeProvider import encry.network.PeerConnectionHandler.{AwaitingHandshake, CommunicationState, _} import encry.network.PeerConnectionHandler.ReceivableMessages._ -import encry.network.PeersKeeper.{ConnectionStopped, HandshakedDone} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, HandshakedDone} import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.{GeneralizedNetworkMessage, Handshake, NetworkMessage} import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index a6e380fc9e..dc92e76c20 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -11,11 +11,11 @@ import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.{LastUptime, PeerInfo} -import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler._ import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus @@ -25,287 +25,287 @@ import org.encryfoundation.common.network.BasicMessagesRepo._ import scala.concurrent.duration._ import scala.util.{Random, Try} - -class PeersKeeper(settings: EncryAppSettings, - nodeViewSync: ActorRef, - dataHolder: ActorRef) extends Actor with StrictLogging { - - import context.dispatcher - - val connectWithOnlyKnownPeers: Boolean = settings.network.connectOnlyWithKnownPeers.getOrElse(true) - - var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() - - var blackList: BlackList = BlackList(settings) - - var knownPeers: Set[InetAddress] = settings.network.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet - - //todo behaviour is incorrect while outgoing connection with connectWithOnlyKnownPeers param - var peersForConnection: Map[InetSocketAddress, Int] = settings.network.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - - var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty - - var outgoingConnections: Set[InetSocketAddress] = Set.empty - - override def preStart(): Unit = { - nodeViewSync ! RegisterMessagesHandler(Seq( - PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", - GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" - ), self) - if (!connectWithOnlyKnownPeers) context.system.scheduler.schedule(2.seconds, settings.network.syncInterval)( - self ! SendToNetwork(GetPeersNetworkMessage, SendToRandom) - ) - context.system.scheduler.schedule(600.millis, settings.blackList.cleanupTime){blackList = blackList.cleanupBlackList} - context.system.scheduler.schedule(10.seconds, 5.seconds) (dataHolder ! ConnectedPeersConnectionHelper(connectedPeers)) - context.system.scheduler.schedule(10.seconds, 5.seconds)( - nodeViewSync ! UpdatedPeersCollection(connectedPeers.collect(getAllPeers, getPeersForDM).toMap) - ) - context.system.eventStream.subscribe(self, classOf[PeerCommandHelper]) - context.system.scheduler.schedule(5.seconds, 5.seconds){ - dataHolder ! UpdatingPeersInfo( - peersForConnection.keys.toSeq, - connectedPeers.collect(getAllPeers, getConnectedPeers), - blackList.getAll - ) - } - } - - override def receive: Receive = workingBehaviour(isBlockChainSynced = false) - - def workingBehaviour(isBlockChainSynced: Boolean): Receive = setupConnectionsLogic - .orElse(networkMessagesProcessingLogic) - .orElse(banPeersLogic) - .orElse(additionalMessages(isBlockChainSynced)) - - def setupConnectionsLogic: Receive = { - case RequestPeerForConnection if connectedPeers.size < settings.network.maxConnections => - def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r - logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + - s"so peer keeper allows to add one more connection. Current available peers are: " + - s"${peersForConnection.mkString(",")}. Current black list is: ${ - blackList.collect((_, _, _, _) => true, mapReason).mkString(",") - }. Current known peers: ${knownPeers.mkString(",")}.") - logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") - logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") - val peers = peersForConnection - .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) - logger.info(s"peers size: ${peers.size}") - Random.shuffle(peers.toSeq) - .headOption - .foreach { case (peer, _) => - outgoingConnections += peer - logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + - s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + - s"${outgoingConnections.mkString(",")}.") - sender() ! PeerForConnection(peer) - awaitingHandshakeConnections += peer - logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + - s" Current is: ${awaitingHandshakeConnections.mkString(",")}") - } - - case RequestPeerForConnection => - logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") - - case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => - logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + - s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + - s"Current known peers: ${knownPeers.mkString(",")}") - val notConnectedYet: Boolean = !connectedPeers.contains(remote) - val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) - if (notConnectedYet && notBannedPeer) { - logger.info(s"Peer: $remote is available to setup connect with.") - if (outgoingConnections.contains(remote)) { - logger.info(s"Got outgoing connection.") - outgoingConnections -= remote - sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) - } - else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { - logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") - sender() ! ConnectionVerified(remote, remoteConnection, Incoming) - } - else if (connectWithOnlyKnownPeers) - logger.info(s"Got incoming connection but we can connect only with known peers.") - else { - logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") - sender() ! ConnectionVerified(remote, remoteConnection, Incoming) - } - } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + - s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") - - case NewConnection(remote, remoteConnection) => - logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + - s"bigger than possible or isSelf: ${isSelf(remote)}.") - - case HandshakedDone(connectedPeer) => - logger.info(s"Peers keeper got approvement about finishing a handshake." + - s" Initializing new peer: ${connectedPeer.socketAddress}") - connectedPeers = connectedPeers.initializePeer(connectedPeer) - logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + - s"${awaitingHandshakeConnections.mkString(",")}.") - awaitingHandshakeConnections -= connectedPeer.socketAddress - peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) - logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + - s" Current collection is: ${peersForConnection.keys.mkString(",")}.") - - case ConnectionStopped(peer) => - logger.info(s"Connection stopped for: $peer.") - awaitingHandshakeConnections -= peer - connectedPeers = connectedPeers.removePeer(peer) - if (blackList.contains(peer.getAddress)) { - peersForConnection -= peer - logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + - s"Current is: ${peersForConnection.mkString(",")}.") - } - - case OutgoingConnectionFailed(peer) => - logger.info(s"Connection failed for: $peer.") - outgoingConnections -= peer - awaitingHandshakeConnections -= peer - val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 - if (connectionAttempts >= settings.network.maxNumberOfReConnections) { - logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") - //todo think about penalty for the less time than general ban - //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) - peersForConnection -= peer - } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) - } - - def networkMessagesProcessingLogic: Receive = { - case DataFromPeer(message, remote) => message match { - case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => - logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") - peers - .filterNot { p => - blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) - }.foreach { p => - logger.info(s"Found new peer: $p. Adding it to the available peers collection.") - peersForConnection = peersForConnection.updated(p, 0) - } - logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") - - case PeersNetworkMessage(_) => - logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + - s"so ignore this message and ban this peer.") - self ! BanPeer(remote, SentPeersMessageWithoutRequest) - - case GetPeersNetworkMessage => - def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = - Try { - if (remote.socketAddress.getAddress.isSiteLocalAddress) true - else add.getAddress.isSiteLocalAddress && add != remote.socketAddress - }.getOrElse(false) - - val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) - logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") - logger.info(s"Remote is side local: ${remote.socketAddress} : ${Try(remote.socketAddress.getAddress.isSiteLocalAddress)}") - remote.handlerRef ! PeersNetworkMessage(peers) - } - } - - def additionalMessages(isBlockChainSynced: Boolean): Receive = { - case OtherNodeSyncingStatus(remote, comparison, _) => - connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote.socketAddress -> comparison)) - - case AccumulatedPeersStatistic(statistic) => - connectedPeers = connectedPeers.updatePriorityStatus(statistic) - - case SendToNetwork(message, strategy) => - val peers: Seq[ConnectedPeer] = connectedPeers.collect(getAllPeers, getConnectedPeers) - strategy.choose(peers).foreach { peer => - logger.debug(s"Sending message: ${message.messageName} to: ${peer.socketAddress}.") - peer.handlerRef ! message - } - - case SendLocalSyncInfo => - logger.debug(s"Received SendLocalSyncInfo from $sender on PK") - val peersWithHP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(HighPriority), getConnectedPeers) - val peersWithIP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(InitialPriority), getConnectedPeers) - - val accumulatedHPPeers = accumulatePeersForSync(peersWithHP, isBlockChainSynced) - val accumulatedIPPeers = accumulatePeersForSync(peersWithIP, isBlockChainSynced) - val accumulatedPeers = accumulatedHPPeers ++: accumulatedIPPeers - - accumulatedPeers.foreach { p => - logger.debug(s"Update uptime from $p") - connectedPeers = connectedPeers.updateLastUptime(Map(p.socketAddress -> LastUptime(System.currentTimeMillis()))) - } - nodeViewSync ! PeersForSyncInfo(accumulatedPeers) - - context.system.scheduler.scheduleOnce(settings.network.syncInterval) { - logger.debug("Scheduler once for SendLocalSyncInfo triggered") - self ! SendLocalSyncInfo - } - - case PeerFromCli(peer) => - if (!blackList.contains(peer.getAddress) && !peersForConnection.contains(peer) && !connectedPeers.contains(peer) && !isSelf(peer)) { - peersForConnection += (peer -> 0) - knownPeers += peer.getAddress - logger.info(s"Added peer: $peer to known peers. Current newPeers are: ${peersForConnection.mkString(",")}." + - s" Current known peers are: ${knownPeers.mkString(",")}.") - } - - case RemovePeerFromBlackList(peer) => blackList = blackList.remove(peer.getAddress) - - case FullBlockChainIsSynced => - logger.info(s"Peers keeper got message: FullBlockChainIsSynced") - context.become(workingBehaviour(isBlockChainSynced = true)) - - case msg => logger.info(s"Peers keeper got unhandled message: $msg.") - } - - def banPeersLogic: Receive = { - case BanPeer(peer, reason) => - logger.info(s"Banning peer: ${peer.socketAddress} for $reason.") - blackList = blackList.banPeer(reason, peer.socketAddress.getAddress) - peer.handlerRef ! CloseConnection - - case BanPeerFromAPI(peer, reason) => - logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") - blackList = blackList.banPeer(reason, peer.getAddress) - } - - //todo NPE in InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) - def isSelf(address: InetSocketAddress): Boolean = Try(address == settings.network.bindAddress || - settings.network.declaredAddress.contains(address) || - InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || - InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) - - def filterByPriority(priority: PeersPriorityStatus)(address: InetSocketAddress, info: PeerInfo): Boolean = { - val isTimeRangeConserved: Boolean = (System.currentTimeMillis() - info.lastUptime.time) > settings.network.syncInterval.toMillis - val isNecessaryPriority: Boolean = info.peerPriorityStatus == priority - logger.debug(s"findByPriorityForSync: peer: $address, isTimeRangeConserved: $isTimeRangeConserved," + - s" isNecessaryPriority: $isNecessaryPriority") - isTimeRangeConserved && isNecessaryPriority - } - - def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer - - def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add - - def getPeersForDM(address: InetSocketAddress, info: PeerInfo): (InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)) = - address -> (info.connectedPeer, info.historyComparisonResult, info.peerPriorityStatus) - - def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true - - def accumulatePeersForSync(peers: Seq[ConnectedPeer], isChainSynced: Boolean): Seq[ConnectedPeer] = peers match { - case coll: Seq[_] if coll.nonEmpty && isChainSynced => - logger.info(s"Peers collection for sync info non empty and block chain is synced. Sending to DM" + - s" peers collection: ${coll.mkString(",")}.") - coll - case coll: Seq[_] if coll.nonEmpty => scala.util.Random.shuffle(coll).headOption.toSeq.map { p => - logger.info(s"Peers collection for sync info non empty but block chain is not synced. Sending to DM" + - s" peer for sync: $p.") - p - } - case _ => - logger.info(s"Peers collection for sync info message is empty.") - Seq.empty[ConnectedPeer] - - } -} - +// +//class PeersKeeper(settings: EncryAppSettings, +// nodeViewSync: ActorRef, +// dataHolder: ActorRef) extends Actor with StrictLogging { +// +// import context.dispatcher +// +// val connectWithOnlyKnownPeers: Boolean = settings.network.connectOnlyWithKnownPeers.getOrElse(true) +// +// var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() +// +// var blackList: BlackList = BlackList(settings) +// +// var knownPeers: Set[InetAddress] = settings.network.knownPeers +// .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet +// +// //todo behaviour is incorrect while outgoing connection with connectWithOnlyKnownPeers param +// var peersForConnection: Map[InetSocketAddress, Int] = settings.network.knownPeers +// .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap +// +// var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty +// +// var outgoingConnections: Set[InetSocketAddress] = Set.empty +// +// override def preStart(): Unit = { +// nodeViewSync ! RegisterMessagesHandler(Seq( +// PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", +// GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" +// ), self) +// if (!connectWithOnlyKnownPeers) context.system.scheduler.schedule(2.seconds, settings.network.syncInterval)( +// self ! SendToNetwork(GetPeersNetworkMessage, SendToRandom) +// ) +// context.system.scheduler.schedule(600.millis, settings.blackList.cleanupTime){blackList = blackList.cleanupBlackList} +// context.system.scheduler.schedule(10.seconds, 5.seconds) (dataHolder ! ConnectedPeersConnectionHelper(connectedPeers)) +// context.system.scheduler.schedule(10.seconds, 5.seconds)( +// nodeViewSync ! UpdatedPeersCollection(connectedPeers.collect(getAllPeers, getPeersForDM).toMap) +// ) +// context.system.eventStream.subscribe(self, classOf[PeerCommandHelper]) +// context.system.scheduler.schedule(5.seconds, 5.seconds){ +// dataHolder ! UpdatingPeersInfo( +// peersForConnection.keys.toSeq, +// connectedPeers.collect(getAllPeers, getConnectedPeers), +// blackList.getAll +// ) +// } +// } +// +// override def receive: Receive = workingBehaviour(isBlockChainSynced = false) +// +// def workingBehaviour(isBlockChainSynced: Boolean): Receive = setupConnectionsLogic +// .orElse(networkMessagesProcessingLogic) +// .orElse(banPeersLogic) +// .orElse(additionalMessages(isBlockChainSynced)) +// +// def setupConnectionsLogic: Receive = { +// case RequestPeerForConnection if connectedPeers.size < settings.network.maxConnections => +// def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r +// logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + +// s"so peer keeper allows to add one more connection. Current available peers are: " + +// s"${peersForConnection.mkString(",")}. Current black list is: ${ +// blackList.collect((_, _, _, _) => true, mapReason).mkString(",") +// }. Current known peers: ${knownPeers.mkString(",")}.") +// logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") +// logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") +// val peers = peersForConnection +// .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) +// logger.info(s"peers size: ${peers.size}") +// Random.shuffle(peers.toSeq) +// .headOption +// .foreach { case (peer, _) => +// outgoingConnections += peer +// logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + +// s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + +// s"${outgoingConnections.mkString(",")}.") +// sender() ! PeerForConnection(peer) +// awaitingHandshakeConnections += peer +// logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + +// s" Current is: ${awaitingHandshakeConnections.mkString(",")}") +// } +// +// case RequestPeerForConnection => +// logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") +// +// case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => +// logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + +// s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + +// s"Current known peers: ${knownPeers.mkString(",")}") +// val notConnectedYet: Boolean = !connectedPeers.contains(remote) +// val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) +// if (notConnectedYet && notBannedPeer) { +// logger.info(s"Peer: $remote is available to setup connect with.") +// if (outgoingConnections.contains(remote)) { +// logger.info(s"Got outgoing connection.") +// outgoingConnections -= remote +// sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) +// } +// else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { +// logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") +// sender() ! ConnectionVerified(remote, remoteConnection, Incoming) +// } +// else if (connectWithOnlyKnownPeers) +// logger.info(s"Got incoming connection but we can connect only with known peers.") +// else { +// logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") +// sender() ! ConnectionVerified(remote, remoteConnection, Incoming) +// } +// } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + +// s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") +// +// case NewConnection(remote, remoteConnection) => +// logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + +// s"bigger than possible or isSelf: ${isSelf(remote)}.") +// +// case HandshakedDone(connectedPeer) => +// logger.info(s"Peers keeper got approvement about finishing a handshake." + +// s" Initializing new peer: ${connectedPeer.socketAddress}") +// connectedPeers = connectedPeers.initializePeer(connectedPeer) +// logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + +// s"${awaitingHandshakeConnections.mkString(",")}.") +// awaitingHandshakeConnections -= connectedPeer.socketAddress +// peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) +// logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + +// s" Current collection is: ${peersForConnection.keys.mkString(",")}.") +// +// case ConnectionStopped(peer) => +// logger.info(s"Connection stopped for: $peer.") +// awaitingHandshakeConnections -= peer +// connectedPeers = connectedPeers.removePeer(peer) +// if (blackList.contains(peer.getAddress)) { +// peersForConnection -= peer +// logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + +// s"Current is: ${peersForConnection.mkString(",")}.") +// } +// +// case OutgoingConnectionFailed(peer) => +// logger.info(s"Connection failed for: $peer.") +// outgoingConnections -= peer +// awaitingHandshakeConnections -= peer +// val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 +// if (connectionAttempts >= settings.network.maxNumberOfReConnections) { +// logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") +// //todo think about penalty for the less time than general ban +// //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) +// peersForConnection -= peer +// } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) +// } +// +// def networkMessagesProcessingLogic: Receive = { +// case DataFromPeer(message, remote) => message match { +// case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => +// logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") +// peers +// .filterNot { p => +// blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) +// }.foreach { p => +// logger.info(s"Found new peer: $p. Adding it to the available peers collection.") +// peersForConnection = peersForConnection.updated(p, 0) +// } +// logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") +// +// case PeersNetworkMessage(_) => +// logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + +// s"so ignore this message and ban this peer.") +// self ! BanPeer(remote, SentPeersMessageWithoutRequest) +// +// case GetPeersNetworkMessage => +// def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = +// Try { +// if (remote.socketAddress.getAddress.isSiteLocalAddress) true +// else add.getAddress.isSiteLocalAddress && add != remote.socketAddress +// }.getOrElse(false) +// +// val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) +// logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") +// logger.info(s"Remote is side local: ${remote.socketAddress} : ${Try(remote.socketAddress.getAddress.isSiteLocalAddress)}") +// remote.handlerRef ! PeersNetworkMessage(peers) +// } +// } +// +// def additionalMessages(isBlockChainSynced: Boolean): Receive = { +// case OtherNodeSyncingStatus(remote, comparison, _) => +// connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote.socketAddress -> comparison)) +// +// case AccumulatedPeersStatistic(statistic) => +// connectedPeers = connectedPeers.updatePriorityStatus(statistic) +// +// case SendToNetwork(message, strategy) => +// val peers: Seq[ConnectedPeer] = connectedPeers.collect(getAllPeers, getConnectedPeers) +// strategy.choose(peers).foreach { peer => +// logger.debug(s"Sending message: ${message.messageName} to: ${peer.socketAddress}.") +// peer.handlerRef ! message +// } +// +// case SendLocalSyncInfo => +// logger.debug(s"Received SendLocalSyncInfo from $sender on PK") +// val peersWithHP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(HighPriority), getConnectedPeers) +// val peersWithIP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(InitialPriority), getConnectedPeers) +// +// val accumulatedHPPeers = accumulatePeersForSync(peersWithHP, isBlockChainSynced) +// val accumulatedIPPeers = accumulatePeersForSync(peersWithIP, isBlockChainSynced) +// val accumulatedPeers = accumulatedHPPeers ++: accumulatedIPPeers +// +// accumulatedPeers.foreach { p => +// logger.debug(s"Update uptime from $p") +// connectedPeers = connectedPeers.updateLastUptime(Map(p.socketAddress -> LastUptime(System.currentTimeMillis()))) +// } +// nodeViewSync ! PeersForSyncInfo(accumulatedPeers) +// +// context.system.scheduler.scheduleOnce(settings.network.syncInterval) { +// logger.debug("Scheduler once for SendLocalSyncInfo triggered") +// self ! SendLocalSyncInfo +// } +// +// case PeerFromCli(peer) => +// if (!blackList.contains(peer.getAddress) && !peersForConnection.contains(peer) && !connectedPeers.contains(peer) && !isSelf(peer)) { +// peersForConnection += (peer -> 0) +// knownPeers += peer.getAddress +// logger.info(s"Added peer: $peer to known peers. Current newPeers are: ${peersForConnection.mkString(",")}." + +// s" Current known peers are: ${knownPeers.mkString(",")}.") +// } +// +// case RemovePeerFromBlackList(peer) => blackList = blackList.remove(peer.getAddress) +// +// case FullBlockChainIsSynced => +// logger.info(s"Peers keeper got message: FullBlockChainIsSynced") +// context.become(workingBehaviour(isBlockChainSynced = true)) +// +// case msg => logger.info(s"Peers keeper got unhandled message: $msg.") +// } +// +// def banPeersLogic: Receive = { +// case BanPeer(peer, reason) => +// logger.info(s"Banning peer: ${peer.socketAddress} for $reason.") +// blackList = blackList.banPeer(reason, peer.socketAddress.getAddress) +// peer.handlerRef ! CloseConnection +// +// case BanPeerFromAPI(peer, reason) => +// logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") +// blackList = blackList.banPeer(reason, peer.getAddress) +// } +// +// //todo NPE in InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) +// def isSelf(address: InetSocketAddress): Boolean = Try(address == settings.network.bindAddress || +// settings.network.declaredAddress.contains(address) || +// InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || +// InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) +// +// def filterByPriority(priority: PeersPriorityStatus)(address: InetSocketAddress, info: PeerInfo): Boolean = { +// val isTimeRangeConserved: Boolean = (System.currentTimeMillis() - info.lastUptime.time) > settings.network.syncInterval.toMillis +// val isNecessaryPriority: Boolean = info.peerPriorityStatus == priority +// logger.debug(s"findByPriorityForSync: peer: $address, isTimeRangeConserved: $isTimeRangeConserved," + +// s" isNecessaryPriority: $isNecessaryPriority") +// isTimeRangeConserved && isNecessaryPriority +// } +// +// def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer +// +// def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add +// +// def getPeersForDM(address: InetSocketAddress, info: PeerInfo): (InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)) = +// address -> (info.connectedPeer, info.historyComparisonResult, info.peerPriorityStatus) +// +// def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true +// +// def accumulatePeersForSync(peers: Seq[ConnectedPeer], isChainSynced: Boolean): Seq[ConnectedPeer] = peers match { +// case coll: Seq[_] if coll.nonEmpty && isChainSynced => +// logger.info(s"Peers collection for sync info non empty and block chain is synced. Sending to DM" + +// s" peers collection: ${coll.mkString(",")}.") +// coll +// case coll: Seq[_] if coll.nonEmpty => scala.util.Random.shuffle(coll).headOption.toSeq.map { p => +// logger.info(s"Peers collection for sync info non empty but block chain is not synced. Sending to DM" + +// s" peer for sync: $p.") +// p +// } +// case _ => +// logger.info(s"Peers collection for sync info message is empty.") +// Seq.empty[ConnectedPeer] +// +// } +//} +// object PeersKeeper { sealed trait PeerCommandHelper @@ -345,9 +345,9 @@ object PeersKeeper { case object GetKnownPeers - def props(settings: EncryAppSettings, - nodeViewSync: ActorRef, - dataHolder: ActorRef): Props = Props(new PeersKeeper(settings, nodeViewSync, dataHolder)) +// def props(settings: EncryAppSettings, +// nodeViewSync: ActorRef, +// dataHolder: ActorRef): Props = Props(new PeersKeeper(settings, nodeViewSync, dataHolder)) class PeersKeeperPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index 77ba1eea69..d71faf7d77 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -14,100 +14,100 @@ import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -class BlackListTests extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - val knowPeersSettings = testNetSettings.copy( - network = settings.network.copy( - knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), - connectOnlyWithKnownPeers = Some(true) - ), - blackList = settings.blackList.copy( - banTime = 2 seconds, - cleanupTime = 3 seconds - )) - - /* - Unit tests - */ - "Black list" should { - "temporary ban requested peer correctly" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) - newBL.contains(peer) shouldBe true - } - "clean black list from peers with expired ban time which were banned by temporary ban" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) - Thread.sleep(2000) - val newBL1 = newBL.cleanupBlackList - newBL1.contains(peer) shouldBe false - } - "don't remove peer from black list before ban time expired" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SentInvForPayload, peer) - val newBL1 = newBL.cleanupBlackList - newBL1.contains(peer) shouldBe true - } - } - - /* - Akka tests - */ - "Peers keeper" should { - "handle ban peer message correctly" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - peersKeeper ! BanPeer(connectedPeer, SpamSender) - peerHandler.expectMsg(CloseConnection) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true - } - "cleanup black list by scheduler correctly" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) - Thread.sleep(6000) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false - } - "don't remove peer from black list before ban time expired" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - Thread.sleep(4000) - peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) - Thread.sleep(2000) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true - } - } -} \ No newline at end of file +//class BlackListTests extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// val knowPeersSettings = testNetSettings.copy( +// network = settings.network.copy( +// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), +// connectOnlyWithKnownPeers = Some(true) +// ), +// blackList = settings.blackList.copy( +// banTime = 2 seconds, +// cleanupTime = 3 seconds +// )) +// +// /* +// Unit tests +// */ +// "Black list" should { +// "temporary ban requested peer correctly" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) +// newBL.contains(peer) shouldBe true +// } +// "clean black list from peers with expired ban time which were banned by temporary ban" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) +// Thread.sleep(2000) +// val newBL1 = newBL.cleanupBlackList +// newBL1.contains(peer) shouldBe false +// } +// "don't remove peer from black list before ban time expired" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SentInvForPayload, peer) +// val newBL1 = newBL.cleanupBlackList +// newBL1.contains(peer) shouldBe true +// } +// } +// +// /* +// Akka tests +// */ +// "Peers keeper" should { +// "handle ban peer message correctly" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// peersKeeper ! BanPeer(connectedPeer, SpamSender) +// peerHandler.expectMsg(CloseConnection) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true +// } +// "cleanup black list by scheduler correctly" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) +// Thread.sleep(6000) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false +// } +// "don't remove peer from black list before ban time expired" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// Thread.sleep(4000) +// peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) +// Thread.sleep(2000) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index bdb2126515..5f8160f949 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -14,28 +14,28 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, PeersNet import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -class ConnectWithNewPeerTests extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - val knowPeersSettings = testNetSettings.copy( - network = testNetSettings.network.copy( - knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), - connectOnlyWithKnownPeers = Some(true) - ), - blackList = testNetSettings.blackList.copy( - banTime = 2 seconds, - cleanupTime = 3 seconds - )) - - "Peers keeper" should { +//class ConnectWithNewPeerTests extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// val knowPeersSettings = testNetSettings.copy( +// network = testNetSettings.network.copy( +// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), +// connectOnlyWithKnownPeers = Some(true) +// ), +// blackList = testNetSettings.blackList.copy( +// banTime = 2 seconds, +// cleanupTime = 3 seconds +// )) +// +// "Peers keeper" should { // "maintain outgoing connection process correctly" in { // /* Request first peer while current number of connections is 0 */ // val networkController = TestProbe() @@ -160,185 +160,185 @@ class ConnectWithNewPeerTests extends WordSpecLike // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false // peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true // } - "remove peer from available if it has been banned" in { - val networkController = TestProbe() - val nodeViewSync = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) - - val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection - - val peerHandler = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - - networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) - networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) - - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - } - "filter peers from network message" in { - val networkController = TestProbe() - val nodeViewSync = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) - - val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection - - val peerHandler = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - - val newPeerHandler = TestProbe() - val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) - - networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) - networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) - networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - - val peer = new InetSocketAddress("172.16.28.98", 9023) - val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) - - networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true - } - "handle successful connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle stop connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true - - peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle failed connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer doesn't contains in black list and connected peers collection" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer contain in black list" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(remoteAddress), System.currentTimeMillis())) - - peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer contain in connected peers" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(remoteAddress), System.currentTimeMillis())) - - peersKeeper ! HandshakedDone(connectedPeer) - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers true" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while peer is equal to local address" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection( - new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle outgoing connection" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - peersKeeper ! RequestPeerForConnection - peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) - networkController.expectMsg( - ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) - } - } -} \ No newline at end of file +// "remove peer from available if it has been banned" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection +// +// val peerHandler = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) +// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) +// +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// } +// "filter peers from network message" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection +// +// val peerHandler = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) +// +// val newPeerHandler = TestProbe() +// val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) +// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) +// networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// +// val peer = new InetSocketAddress("172.16.28.98", 9023) +// val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) +// +// networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true +// } +// "handle successful connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle stop connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle failed connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer doesn't contains in black list and connected peers collection" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer contain in black list" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(remoteAddress), System.currentTimeMillis())) +// +// peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer contain in connected peers" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(remoteAddress), System.currentTimeMillis())) +// +// peersKeeper ! HandshakedDone(connectedPeer) +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers true" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while peer is equal to local address" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection( +// new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle outgoing connection" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// peersKeeper ! RequestPeerForConnection +// peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true +// networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 0bb95839cc..03d54c59cb 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -17,41 +17,41 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.mutable import scala.collection.mutable.WrappedArray -object DMUtils extends InstanceFactory { - - def initialiseDeliveryManager(isBlockChainSynced: Boolean, - isMining: Boolean, - settings: EncryAppSettings) - (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { - val history: History = generateDummyHistory(settings) - val deliveryManager: TestActorRef[DeliveryManager] = - TestActorRef[DeliveryManager](DeliveryManager - .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) - deliveryManager ! UpdatedHistory(history) - if (isMining) deliveryManager ! StartMining - else deliveryManager ! DisableMining - if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced - (deliveryManager, history) - } - - def generateBlocks(qty: Int, history: History): (History, List[Block]) = - (0 until qty).foldLeft(history, List.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - val a = prevHistory.reportModifierIsValid(block) - (a, blocks :+ block) - } - - def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - - def createPeer(port: Int, - host: String, - settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { - val address = new InetSocketAddress(host, port) - val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, - Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) - (address, peer) - } -} \ No newline at end of file +//object DMUtils extends InstanceFactory { +// +// def initialiseDeliveryManager(isBlockChainSynced: Boolean, +// isMining: Boolean, +// settings: EncryAppSettings) +// (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { +// val history: History = generateDummyHistory(settings) +// val deliveryManager: TestActorRef[DeliveryManager] = +// TestActorRef[DeliveryManager](DeliveryManager +// .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) +// deliveryManager ! UpdatedHistory(history) +// if (isMining) deliveryManager ! StartMining +// else deliveryManager ! DisableMining +// if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced +// (deliveryManager, history) +// } +// +// def generateBlocks(qty: Int, history: History): (History, List[Block]) = +// (0 until qty).foldLeft(history, List.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// val a = prevHistory.reportModifierIsValid(block) +// (a, blocks :+ block) +// } +// +// def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) +// +// def createPeer(port: Int, +// host: String, +// settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { +// val address = new InetSocketAddress(host, port) +// val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) +// (address, peer) +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala index 410373e2b3..fb358c98b8 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala @@ -2,7 +2,7 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} +//import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit} import encry.consensus.HistoryConsensus @@ -10,7 +10,7 @@ import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal +//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PeersKeeper.UpdatedPeersCollection @@ -21,225 +21,225 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -class DeliveryManagerPriorityTests extends WordSpecLike - with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll: Unit = TestKit.shutdownActorSystem(system) - - def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, - ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, - List[Block], List[ModifierId]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val (_: InetSocketAddress, cp4: ConnectedPeer) = createPeer(9004, "172.16.13.13", testNetSettings) - val (_: InetSocketAddress, cp5: ConnectedPeer) = createPeer(9005, "172.16.13.14", testNetSettings) - val (_: InetSocketAddress, cp6: ConnectedPeer) = createPeer(9006, "172.16.13.15", testNetSettings) - val (_: InetSocketAddress, cp7: ConnectedPeer) = createPeer(9007, "172.16.13.16", testNetSettings) - val (_: InetSocketAddress, cp8: ConnectedPeer) = createPeer(9008, "172.16.13.17", testNetSettings) - val (_: InetSocketAddress, cp9: ConnectedPeer) = createPeer(9009, "172.16.13.18", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6,cp7, cp8, cp9, blocks, headersIds) - } - - "Delivery Manager" should { - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority. - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1 for cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive less than 1/2 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as BadNode. - * - */ - "mark peer as BadNode with BadPriority (1)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == BadNode) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 3\4 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as BestNode. - */ - "mark peer as HighPriorityNode with HighPriority (4)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == HighPriority) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 1\2 and less than 3\4 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as LowPriorityNode. - */ - "mark peer as LowPriorityNode with LowPriority (3)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(6).map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == LowPriority) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behavior connected with updating several nodes priority active in one time - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1, cp2, cp3, cp4, cp5, cp6. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1, cp2, cp3, cp4, cp5, cp6. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 3\4 requested modifiers from cp1 and cp4. - * Receive less than 3\4 but more than 1\2 requested modifiers from cp2 and cp5. - * Receive less than 1\2 requested modifiers from cp3 and cp6. - * When period of updating priorities will expire, delivery manager will mark cp1 and cp4 as HighPriorityNode. - * When period of updating priorities will expire, delivery manager will mark cp2 and cp5 as LowPriorityNode. - * When period of updating priorities will expire, delivery manager will mark cp3 and cp6 as BadNode. - */ - "correctly choose peer priority while several peers are available" in { - val (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6, cp7, cp8, cp9, blocks, headersIds) = initialiseState - val updatedPeersCollection = - Map( - cp1.socketAddress -> (cp1, Older, InitialPriority), - cp1.socketAddress -> (cp2, Younger, InitialPriority), - cp1.socketAddress -> (cp3, Equal, InitialPriority), - cp1.socketAddress -> (cp4, Older, InitialPriority), - cp1.socketAddress -> (cp5, Younger, InitialPriority), - cp1.socketAddress -> (cp6, Equal, InitialPriority), - cp1.socketAddress -> (cp7, Older, InitialPriority), - cp1.socketAddress -> (cp8, Younger, InitialPriority), - cp1.socketAddress -> (cp9, Equal, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp4, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp5, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp6, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp7, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp8, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp9, Header.modifierTypeId, headersIds) - - val headerBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp3) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp4) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp5) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp6) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp7) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp8) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp9) - - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == HighPriority) - - //todo fix spam after it fix test -// assert(result.contains(cp2.socketAddress)) -// assert(result(cp2.socketAddress) == HighPriority()) - -// assert(result.contains(cp3.socketAddress)) -// assert(result(cp3.socketAddress) == HighPriority()) - -// assert(result.contains(cp4.socketAddress)) -// assert(result(cp4.socketAddress) == LowPriority()) +//class DeliveryManagerPriorityTests extends WordSpecLike +// with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { // -// assert(result.contains(cp5.socketAddress)) -// assert(result(cp5.socketAddress) == LowPriority()) +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") // -// assert(result.contains(cp6.socketAddress)) -// assert(result(cp6.socketAddress) == LowPriority()) +// override def afterAll: Unit = TestKit.shutdownActorSystem(system) // -// assert(result.contains(cp7.socketAddress)) -// assert(result(cp7.socketAddress) == BadNode()) +// def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, +// ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, +// List[Block], List[ModifierId]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val (_: InetSocketAddress, cp4: ConnectedPeer) = createPeer(9004, "172.16.13.13", testNetSettings) +// val (_: InetSocketAddress, cp5: ConnectedPeer) = createPeer(9005, "172.16.13.14", testNetSettings) +// val (_: InetSocketAddress, cp6: ConnectedPeer) = createPeer(9006, "172.16.13.15", testNetSettings) +// val (_: InetSocketAddress, cp7: ConnectedPeer) = createPeer(9007, "172.16.13.16", testNetSettings) +// val (_: InetSocketAddress, cp8: ConnectedPeer) = createPeer(9008, "172.16.13.17", testNetSettings) +// val (_: InetSocketAddress, cp9: ConnectedPeer) = createPeer(9009, "172.16.13.18", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6,cp7, cp8, cp9, blocks, headersIds) +// } // -// assert(result.contains(cp8.socketAddress)) -// assert(result(cp8.socketAddress) == BadNode()) +// "Delivery Manager" should { +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority. +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1 for cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive less than 1/2 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as BadNode. +// * +// */ +// "mark peer as BadNode with BadPriority (1)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == BadNode) +// deliveryManager.stop() +// } // -// assert(result.contains(cp9.socketAddress)) -// assert(result(cp9.socketAddress) == BadNode()) - - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behavior connected with updating node priority while receiving spam modifiers - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Receive unexpected modifiers from cp1. - * cp1 priority must stay as InitialPriority. - */ - "not increment modifiers which will be putted in spam collection" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, _) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == BadNode) - deliveryManager.stop() - } - } -} \ No newline at end of file +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 3\4 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as BestNode. +// */ +// "mark peer as HighPriorityNode with HighPriority (4)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == HighPriority) +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 1\2 and less than 3\4 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as LowPriorityNode. +// */ +// "mark peer as LowPriorityNode with LowPriority (3)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(6).map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == LowPriority) +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behavior connected with updating several nodes priority active in one time +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1, cp2, cp3, cp4, cp5, cp6. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1, cp2, cp3, cp4, cp5, cp6. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 3\4 requested modifiers from cp1 and cp4. +// * Receive less than 3\4 but more than 1\2 requested modifiers from cp2 and cp5. +// * Receive less than 1\2 requested modifiers from cp3 and cp6. +// * When period of updating priorities will expire, delivery manager will mark cp1 and cp4 as HighPriorityNode. +// * When period of updating priorities will expire, delivery manager will mark cp2 and cp5 as LowPriorityNode. +// * When period of updating priorities will expire, delivery manager will mark cp3 and cp6 as BadNode. +// */ +// "correctly choose peer priority while several peers are available" in { +// val (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6, cp7, cp8, cp9, blocks, headersIds) = initialiseState +// val updatedPeersCollection = +// Map( +// cp1.socketAddress -> (cp1, Older, InitialPriority), +// cp1.socketAddress -> (cp2, Younger, InitialPriority), +// cp1.socketAddress -> (cp3, Equal, InitialPriority), +// cp1.socketAddress -> (cp4, Older, InitialPriority), +// cp1.socketAddress -> (cp5, Younger, InitialPriority), +// cp1.socketAddress -> (cp6, Equal, InitialPriority), +// cp1.socketAddress -> (cp7, Older, InitialPriority), +// cp1.socketAddress -> (cp8, Younger, InitialPriority), +// cp1.socketAddress -> (cp9, Equal, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp4, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp5, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp6, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp7, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp8, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp9, Header.modifierTypeId, headersIds) +// +// val headerBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp3) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp4) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp5) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp6) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp7) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp8) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp9) +// +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == HighPriority) +// +// //todo fix spam after it fix test +//// assert(result.contains(cp2.socketAddress)) +//// assert(result(cp2.socketAddress) == HighPriority()) +// +//// assert(result.contains(cp3.socketAddress)) +//// assert(result(cp3.socketAddress) == HighPriority()) +// +//// assert(result.contains(cp4.socketAddress)) +//// assert(result(cp4.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp5.socketAddress)) +//// assert(result(cp5.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp6.socketAddress)) +//// assert(result(cp6.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp7.socketAddress)) +//// assert(result(cp7.socketAddress) == BadNode()) +//// +//// assert(result.contains(cp8.socketAddress)) +//// assert(result(cp8.socketAddress) == BadNode()) +//// +//// assert(result.contains(cp9.socketAddress)) +//// assert(result(cp9.socketAddress) == BadNode()) +// +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behavior connected with updating node priority while receiving spam modifiers +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Receive unexpected modifiers from cp1. +// * cp1 priority must stay as InitialPriority. +// */ +// "not increment modifiers which will be putted in spam collection" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, _) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == BadNode) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index 97acbe4976..106005b9ff 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -1,187 +1,187 @@ -package encry.network.DeliveryManagerTests - -import java.net.InetSocketAddress -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.Older -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.settings.TestNetSettings -import encry.view.history.History -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scala.concurrent.duration._ -import scala.collection.mutable.WrappedArray - -class DeliveryManagerReRequestModifiesSpec extends WordSpecLike - with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = system.terminate() - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { - val (deliveryManager, history) = - initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) - } - - "ReRequestModifies" should { - "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + - "expectedModifiers collection after all attempts will expire" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: ModifierId = headersIds.head - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) - handler1.expectMsgAllOf( - testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) - ) - //this thread sleep is using for expecting modifier removal - Thread.sleep(6000) - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) - deliveryManager.stop() - } - "not re-ask unnecessary modifiers" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: ModifierId = headersIds.head - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) - - //await one re-ask - handler1.expectMsgAllOf( - testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) - ) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, - Map(header -> headerBytes)), cp1) - deliveryManager.stop() - } - "not re-ask modifiers which were applied to the history" in { - val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - - handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, - Map(headerIds.head -> headerBytes)), cp1) - - history.append(blocks.head.header) - val uHistory: History = history.reportModifierIsValid(blocks.head.header) - - deliveryManager ! UpdatedHistory(uHistory) - - deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty).isEmpty) - deliveryManager.stop() - } - "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { - val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - //this thread sleep is using for expecting modifier removal - Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) - deliveryManager.stop() - } - "not re-ask transactions" in { - val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) - ) - handler1.expectNoMsg(10.seconds) - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) - deliveryManager.stop() - } - } -} \ No newline at end of file +//package encry.network.DeliveryManagerTests +// +//import java.net.InetSocketAddress +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.Older +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +////import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.NodeViewSynchronizer.ReceivableMessages._ +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.settings.TestNetSettings +//import encry.view.history.History +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import scala.concurrent.duration._ +//import scala.collection.mutable.WrappedArray +// +//class DeliveryManagerReRequestModifiesSpec extends WordSpecLike +// with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = system.terminate() +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { +// val (deliveryManager, history) = +// initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) +// } +// +// "ReRequestModifies" should { +// "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + +// "expectedModifiers collection after all attempts will expire" in { +// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: ModifierId = headersIds.head +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// handler1.expectMsgAllOf( +// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) +// ) +// //this thread sleep is using for expecting modifier removal +// Thread.sleep(6000) +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// deliveryManager.stop() +// } +// "not re-ask unnecessary modifiers" in { +// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: ModifierId = headersIds.head +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// +// //await one re-ask +// handler1.expectMsgAllOf( +// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) +// ) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, +// Map(header -> headerBytes)), cp1) +// deliveryManager.stop() +// } +// "not re-ask modifiers which were applied to the history" in { +// val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// +// handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, +// Map(headerIds.head -> headerBytes)), cp1) +// +// history.append(blocks.head.header) +// val uHistory: History = history.reportModifierIsValid(blocks.head.header) +// +// deliveryManager ! UpdatedHistory(uHistory) +// +// deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// deliveryManager.stop() +// } +// "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { +// val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// //this thread sleep is using for expecting modifier removal +// Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// deliveryManager.stop() +// } +// "not re-ask transactions" in { +// val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) +// ) +// handler1.expectNoMsg(10.seconds) +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 190f11c8d4..0ab92b2abf 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,320 +1,320 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.TestNetSettings -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -import encry.view.NodeViewHolder.DownloadRequest -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.collection.mutable.WrappedArray - -class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) - } - - "RequestModifies" should { - "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) - headersIds.foreach(id => - deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => - block.id.sameElements(id)).get.payload.id)) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .size == blocks.size) - deliveryManager.stop() - } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) - - deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers while block chain is not synced from Younger nodes" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection = - Map( - address2 -> (cp2, Younger, InitialPriority), - address3 -> (cp3, Fork, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - handler2.expectNoMsg() - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) - ) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) - assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) - deliveryManager.stop() - } - } -} \ No newline at end of file +// +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.TestNetSettings +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +//import encry.view.NodeViewHolder.DownloadRequest +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +//import org.encryfoundation.common.network.SyncInfo +//import org.encryfoundation.common.utils.Algos +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +// +//import scala.collection.mutable.WrappedArray + +//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) +// } +// +// "RequestModifies" should { +// "handle uniq modifiers from RequestFromLocal message correctly" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "not handle repeating modifiers from RequestFromLocal message" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery Manager should handle received modifier which were requested correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery manager should not handle repeating modifiers" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "handle priority request for payload correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) +// headersIds.foreach(id => +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => +// block.id.sameElements(id)).get.payload.id)) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .size == blocks.size) +// deliveryManager.stop() +// } +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) +// +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers while block chain is not synced from Younger nodes" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection = +// Map( +// address2 -> (cp2, Younger, InitialPriority), +// address3 -> (cp3, Fork, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// handler2.expectNoMsg() +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask transactions while block chain is not synced" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not ask transaction while node is not mining" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not re-ask modifiers which already have been received" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) +// ) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) +// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala index d432fad199..2b8ffd799c 100644 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala @@ -11,7 +11,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, U import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } import encry.network.PeersKeeper.BanPeer import encry.settings.TestNetSettings -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +//import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.History import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.history.{ @@ -27,205 +27,205 @@ import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpec import scorex.crypto.hash.Digest32 import scorex.utils.Random -class DownloadedModifiersValidatorTests - extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - "DownloadedModifiersValidatorTests" should { - "find too old header by height" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) - downloadedModifiersValidator ! UpdatedHistory(history) - val invalidHeader = generateGenesisBlock(Height @@ 1) - - val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) - .map( - b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray - ) - .toMap - import scala.concurrent.duration._ - downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) - peersKeeper.expectMsgPF(10.seconds) { - case BanPeer(connected, _) => connected == connectedPeer - } - val validHeightHeader = generateGenesisBlock(Height @@ 200) - - val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) - .map( - b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray - ) - .toMap - downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) - nodeViewHolder.expectMsgPF(10.seconds) { - case ModifierFromRemote(mod) => mod == validHeightHeader.header - } - } - "find corrupted header" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val deliveryManager = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val history: History = generateDummyHistory(testNetSettings) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val timestamp1 = System.currentTimeMillis() - Thread.sleep(1000) - val timestamp2 = System.currentTimeMillis() - - val header_first: Header = Header( - 1.toByte, - ModifierId @@ Random.randomBytes(), - Digest32 @@ Random.randomBytes(), - timestamp2, - 2, - scala.util.Random.nextLong(), - testNetSettings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)), - Random.randomBytes() - ) - val header_second: Header = Header( - 1.toByte, - header_first.id, - Digest32 @@ Random.randomBytes(), - timestamp1, - 1, - scala.util.Random.nextLong(), - testNetSettings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)), - Random.randomBytes() - ) - - history.append(header_first) - - nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) - - /* Header */ - val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap - val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) - - deliveryManager.send(downloadedModifiersValidator, msg) - peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) - nodeViewHolder.expectNoMsg() - nodeViewSync.expectMsg(InvalidModifier(header_second.id)) - } - "find corrupted payload" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val deliveryManager = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val history: History = generateDummyHistory(testNetSettings) - - val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - (prevHistory.reportModifierIsValid(block), blocks :+ block) - } - - val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) - - nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) - - val bytes = PayloadProtoSerializer.toProto(payload).toByteArray - - val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( - b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse - ) :+ (payload.id -> bytes)).toMap - - deliveryManager - .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) - - peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) - nodeViewHolder.expectMsg(ModifierFromRemote(payload)) - } - } - - def generateBlocks(qty: Int, history: History): (History, List[Block]) = - (0 until qty).foldLeft(history, List.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - val a = prevHistory.reportModifierIsValid(block) - (a, blocks :+ block) - } -} +//class DownloadedModifiersValidatorTests +// extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// "DownloadedModifiersValidatorTests" should { +// "find too old header by height" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) +// downloadedModifiersValidator ! UpdatedHistory(history) +// val invalidHeader = generateGenesisBlock(Height @@ 1) +// +// val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) +// .map( +// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray +// ) +// .toMap +// import scala.concurrent.duration._ +// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) +// peersKeeper.expectMsgPF(10.seconds) { +// case BanPeer(connected, _) => connected == connectedPeer +// } +// val validHeightHeader = generateGenesisBlock(Height @@ 200) +// +// val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) +// .map( +// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray +// ) +// .toMap +// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) +// nodeViewHolder.expectMsgPF(10.seconds) { +// case ModifierFromRemote(mod) => mod == validHeightHeader.header +// } +// } +// "find corrupted header" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val deliveryManager = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val history: History = generateDummyHistory(testNetSettings) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val timestamp1 = System.currentTimeMillis() +// Thread.sleep(1000) +// val timestamp2 = System.currentTimeMillis() +// +// val header_first: Header = Header( +// 1.toByte, +// ModifierId @@ Random.randomBytes(), +// Digest32 @@ Random.randomBytes(), +// timestamp2, +// 2, +// scala.util.Random.nextLong(), +// testNetSettings.constants.InitialDifficulty, +// EquihashSolution(Seq(1, 3)), +// Random.randomBytes() +// ) +// val header_second: Header = Header( +// 1.toByte, +// header_first.id, +// Digest32 @@ Random.randomBytes(), +// timestamp1, +// 1, +// scala.util.Random.nextLong(), +// testNetSettings.constants.InitialDifficulty, +// EquihashSolution(Seq(1, 3)), +// Random.randomBytes() +// ) +// +// history.append(header_first) +// +// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) +// +// /* Header */ +// val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap +// val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) +// +// deliveryManager.send(downloadedModifiersValidator, msg) +// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) +// nodeViewHolder.expectNoMsg() +// nodeViewSync.expectMsg(InvalidModifier(header_second.id)) +// } +// "find corrupted payload" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val deliveryManager = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val history: History = generateDummyHistory(testNetSettings) +// +// val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// (prevHistory.reportModifierIsValid(block), blocks :+ block) +// } +// +// val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) +// +// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) +// +// val bytes = PayloadProtoSerializer.toProto(payload).toByteArray +// +// val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( +// b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse +// ) :+ (payload.id -> bytes)).toMap +// +// deliveryManager +// .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) +// +// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) +// nodeViewHolder.expectMsg(ModifierFromRemote(payload)) +// } +// } +// +// def generateBlocks(qty: Int, history: History): (History, List[Block]) = +// (0 until qty).foldLeft(history, List.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// val a = prevHistory.reportModifierIsValid(block) +// (a, blocks :+ block) +// } +//} diff --git a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala index 265dfff90f..d58e74ce0e 100644 --- a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala +++ b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala @@ -1,120 +1,120 @@ -package encry.view.history - -import encry.consensus.HistoryConsensus._ -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.SyncInfo -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} - -class HistoryComparisionResultTest extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - "History Reader" should { - "mark history as Equal where our best header is the same as other history best header" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Equal) - } - - "mark history as Older where our best header is in other history best chain, but not at the last position" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) - - val updatedHistory: History = blocks.take(50).foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Older) - } - - "mark history as Younger when comparing history is empty" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(Seq.empty) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Younger) - } - - "mark history as Younger when our history contains all other history but other history " + - "doesn't contain our last 70 headers" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.take(30).map(_.header.id)) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Younger) - } - - "mark history as Fork when we have same point in histories" in { - val history: History = generateDummyHistory(testNetSettings) - - val fork = genForkOn(100, 1000, 25, 30, testNetSettings) - - val syncInfo: SyncInfo = SyncInfo( - fork._1.take(25).map(_.header.id) ++: fork._2.map(_.header.id) - ) - - val updatedHistory: History = fork._1.take(30).foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Fork) - } - - "mark history as Equal where both nodes do not keep any blocks" in { - val history: History = generateDummyHistory(testNetSettings) - val syncInfo: SyncInfo = SyncInfo(Seq.empty) - - val comparisonResult = history.compare(syncInfo) - assert(comparisonResult == Equal) - } - - "mark history as Older " in { - val history: History = generateDummyHistory(testNetSettings) - val syncInfo: SyncInfo = SyncInfo( - generateBlocks(30, generateDummyHistory(testNetSettings))._2.map(_.header.id)) - - val comparisonResult = history.compare(syncInfo) - assert(comparisonResult == Older) - } - } -} \ No newline at end of file +//package encry.view.history +// +//import encry.consensus.HistoryConsensus._ +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import org.encryfoundation.common.modifiers.history.Block +//import org.encryfoundation.common.network.SyncInfo +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +// +//class HistoryComparisionResultTest extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// "History Reader" should { +// "mark history as Equal where our best header is the same as other history best header" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Equal) +// } +// +// "mark history as Older where our best header is in other history best chain, but not at the last position" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) +// +// val updatedHistory: History = blocks.take(50).foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Older) +// } +// +// "mark history as Younger when comparing history is empty" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(Seq.empty) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Younger) +// } +// +// "mark history as Younger when our history contains all other history but other history " + +// "doesn't contain our last 70 headers" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.take(30).map(_.header.id)) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Younger) +// } +// +// "mark history as Fork when we have same point in histories" in { +// val history: History = generateDummyHistory(testNetSettings) +// +// val fork = genForkOn(100, 1000, 25, 30, testNetSettings) +// +// val syncInfo: SyncInfo = SyncInfo( +// fork._1.take(25).map(_.header.id) ++: fork._2.map(_.header.id) +// ) +// +// val updatedHistory: History = fork._1.take(30).foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Fork) +// } +// +// "mark history as Equal where both nodes do not keep any blocks" in { +// val history: History = generateDummyHistory(testNetSettings) +// val syncInfo: SyncInfo = SyncInfo(Seq.empty) +// +// val comparisonResult = history.compare(syncInfo) +// assert(comparisonResult == Equal) +// } +// +// "mark history as Older " in { +// val history: History = generateDummyHistory(testNetSettings) +// val syncInfo: SyncInfo = SyncInfo( +// generateBlocks(30, generateDummyHistory(testNetSettings))._2.map(_.header.id)) +// +// val comparisonResult = history.compare(syncInfo) +// assert(comparisonResult == Older) +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/view/history/ModifiersValidationTest.scala b/src/test/scala/encry/view/history/ModifiersValidationTest.scala index e879864ea8..722258dc54 100644 --- a/src/test/scala/encry/view/history/ModifiersValidationTest.scala +++ b/src/test/scala/encry/view/history/ModifiersValidationTest.scala @@ -1,45 +1,45 @@ -package encry.view.history - -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} -import org.encryfoundation.common.modifiers.history.Block -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} - -class ModifiersValidationTest extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - "Modifiers validator" should { - "validate genesis block" in { - val newHistory: History = generateDummyHistory(testNetSettings) - val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) - newHistory.testApplicable(genesisBlock.header).isRight shouldBe true - newHistory.append(genesisBlock.header) - val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) - updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true - } - "reject incorrect modifiers" in { - val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 - val newHistory: History = generateDummyHistory(testNetSettings) - blocks.take(1).foldLeft(newHistory) { case (history, block) => - history.testApplicable(block.header).isRight shouldBe true - history.append(block.header) - history.reportModifierIsValid(block.header) - history.testApplicable(block.payload).isRight shouldBe true - history.append(block.payload) - history.reportModifierIsValid(block) - } - blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => - history.testApplicable(block.header).isRight shouldBe false - history.append(block.header) - history.reportModifierIsValid(block.header) - history.testApplicable(block.payload).isRight shouldBe true - history.append(block.payload) - history.reportModifierIsValid(block) - } - } - } -} \ No newline at end of file +//package encry.view.history +// +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import org.encryfoundation.common.modifiers.history.Block +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +// +//class ModifiersValidationTest extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// "Modifiers validator" should { +// "validate genesis block" in { +// val newHistory: History = generateDummyHistory(testNetSettings) +// val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) +// newHistory.testApplicable(genesisBlock.header).isRight shouldBe true +// newHistory.append(genesisBlock.header) +// val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) +// updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true +// } +// "reject incorrect modifiers" in { +// val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 +// val newHistory: History = generateDummyHistory(testNetSettings) +// blocks.take(1).foldLeft(newHistory) { case (history, block) => +// history.testApplicable(block.header).isRight shouldBe true +// history.append(block.header) +// history.reportModifierIsValid(block.header) +// history.testApplicable(block.payload).isRight shouldBe true +// history.append(block.payload) +// history.reportModifierIsValid(block) +// } +// blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => +// history.testApplicable(block.header).isRight shouldBe false +// history.append(block.header) +// history.reportModifierIsValid(block.header) +// history.testApplicable(block.payload).isRight shouldBe true +// history.append(block.payload) +// history.reportModifierIsValid(block) +// } +// } +// } +//} \ No newline at end of file From dca7c6f516ab7c1a63b773b6a525997833b579b5 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 15:15:47 +0300 Subject: [PATCH 032/119] Merge branch 'EC-859' of /Users/timofey/idea/encry/core with conflicts. --- .../scala/encry/nvg/IntermediaryNVH.scala | 15 +- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 30 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 23 +- .../DeliveryManagerRequestModifiesSpec.scala | 633 +++++++++--------- 5 files changed, 351 insertions(+), 352 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index d338e100fb..9a8beaf0fa 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -7,7 +7,7 @@ import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ @@ -82,17 +82,16 @@ class IntermediaryNVH( networkMessagesProcessor ! newReader case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifier(_) => intermediaryNetwork ! msg - case msg @ FastSyncDone => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _, _) => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ ModifiersNetworkMessage(_, _) => intermediaryNetwork ! msg - case msg @ SendToNetwork(_, _) => intermediaryNetwork ! msg - case msg @ IdsForRequest(_) => intermediaryNetwork ! msg + case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => intermediaryNetwork ! msg - case msg @ FullBlockChainIsSynced => intermediaryNetwork ! msg //+ to miner + case msg @ FastSyncDone => + case msg @ HeaderChainIsSynced => + case msg @ FullBlockChainIsSynced => //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 215f1ec2a2..cb91e6d0d1 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -44,7 +44,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! InvalidModifier(modifierId) + context.parent ! SyntacticallyInvalidPersistentModifier(modifierId) } else preSemanticValidation.leftMap { case IllegalHeight(error) => diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 7ed1c56f3e..767cdbb013 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,23 +1,26 @@ package encry.nvg -import akka.actor.{Actor, Cancellable, Props} +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} -import encry.network.Broadcast +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{OtherNodeSyncingStatus, SemanticallySuccessfulModifier} -import encry.network.PeersKeeper.SendToNetwork -import encry.nvg.NetworkMessagesProcessor.IdsForRequest +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.nvg.NodeViewHolder.DownloadRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -44,7 +47,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case block: Block if historyReader.isFullChainSynced => List(block.header, block.payload).foreach { mod: PersistentModifier => logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") - context.parent ! SendToNetwork(InvNetworkMessage(mod.modifierTypeId -> Seq(mod.id)), Broadcast) + context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) } modifiersRequestCache = Map( block.encodedId -> toProto(block.header), @@ -100,18 +103,18 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St typeId match { case h if h == Header.modifierTypeId => - context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(unrequestedModifiers, historyReader)) + context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) case _ => getModsForRemote(unrequestedModifiers, historyReader).foreach { case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) } } } case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) context.parent ! IdsForRequest(newIds.toList) + if (newIds.nonEmpty) context.parent ! DownloadRequest(Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) context.become(workingCycle(nextCheckModsScheduler.some)) @@ -125,6 +128,5 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } object NetworkMessagesProcessor { - final case class IdsForRequest(ids: List[ModifierId]) extends AnyVal def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index a8f8aae222..60bfe7705c 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,18 +2,18 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.NodeViewHolder.{DownloadRequest, NodeView, UpdateInformation} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -23,7 +23,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -31,14 +31,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -171,7 +171,7 @@ class NodeViewHolder( s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) - context.parent ! DownloadRequest(tid, id, previousModifier) + context.parent ! DownloadRequest(tid, List(id)) else logger.info( s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." @@ -501,8 +501,7 @@ object NodeViewHolder { final case class DownloadRequest( modifierTypeId: ModifierTypeId, - modifierId: ModifierId, - previousModifier: Option[ModifierId] = none + modifierIds: List[ModifierId] ) extends NodeViewHolderEvent final case class UpdateInformation( diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 0ab92b2abf..067b867ecd 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,320 +1,319 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -// -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestKit, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.settings.TestNetSettings -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -//import encry.view.NodeViewHolder.DownloadRequest -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -//import org.encryfoundation.common.network.SyncInfo -//import org.encryfoundation.common.utils.Algos -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -// -//import scala.collection.mutable.WrappedArray - -//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { -// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) -// } -// -// "RequestModifies" should { -// "handle uniq modifiers from RequestFromLocal message correctly" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "not handle repeating modifiers from RequestFromLocal message" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery Manager should handle received modifier which were requested correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery manager should not handle repeating modifiers" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "handle priority request for payload correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) -// headersIds.foreach(id => -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => -// block.id.sameElements(id)).get.payload.id)) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .size == blocks.size) -// deliveryManager.stop() -// } -// "choose correct peer in priority request" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map( -// address1 -> (cp1, Older, InitialPriority), -// address2 -> (cp2, Older, InitialPriority), -// address3 -> (cp3, Older, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) -// -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) -// -// handler1.expectMsgAnyOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), -// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), -// SyncInfoNetworkMessage(SyncInfo(List())) -// ) -// -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers while block chain is not synced from Younger nodes" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection = -// Map( -// address2 -> (cp2, Younger, InitialPriority), -// address3 -> (cp3, Fork, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// handler2.expectNoMsg() -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers from peer which is not contained in status tracker" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address2 -> (cp2, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask transactions while block chain is not synced" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not ask transaction while node is not mining" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not re-ask modifiers which already have been received" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) -// ) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) -// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) -// deliveryManager.stop() -// } -// } -//} \ No newline at end of file + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestKit, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.settings.TestNetSettings +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.collection.mutable.WrappedArray + +class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = TestKit.shutdownActorSystem(system) + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { + val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + } + + "RequestModifies" should { + "handle uniq modifiers from RequestFromLocal message correctly" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "not handle repeating modifiers from RequestFromLocal message" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery Manager should handle received modifier which were requested correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery manager should not handle repeating modifiers" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "handle priority request for payload correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) + headersIds.foreach(id => + deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => + block.id.sameElements(id)).get.payload.id)) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .size == blocks.size) + deliveryManager.stop() + } + "choose correct peer in priority request" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map( + address1 -> (cp1, Older, InitialPriority), + address2 -> (cp2, Older, InitialPriority), + address3 -> (cp3, Older, InitialPriority) + ) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) + + deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) + + handler1.expectMsgAnyOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), + RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), + SyncInfoNetworkMessage(SyncInfo(List())) + ) + + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers while block chain is not synced from Younger nodes" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection = + Map( + address2 -> (cp2, Younger, InitialPriority), + address3 -> (cp3, Fork, InitialPriority) + ) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) + + handler2.expectNoMsg() + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers from peer which is not contained in status tracker" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address2 -> (cp2, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + + handler1.expectNoMsg() + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask transactions while block chain is not synced" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not ask transaction while node is not mining" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not re-ask modifiers which already have been received" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) + + handler1.expectMsgAllOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) + ) + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + + handler1.expectNoMsg() + + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) + assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) + deliveryManager.stop() + } + } +} \ No newline at end of file From 3ad6215d16562d9dd824769e94fa0bec99980797 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 15:25:30 +0300 Subject: [PATCH 033/119] added messages --- .../DownloadedModifiersValidator.scala | 1 - .../scala/encry/nvg/IntermediaryNVH.scala | 46 +++++++++---------- .../scala/encry/nvg/ModifiersValidator.scala | 12 +++-- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 3c4651e727..156bdc9009 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -66,7 +66,6 @@ object DownloadedModifiersValidator { typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) - final case class InvalidModifier(ids: ModifierId) extends AnyVal // def props(modifierIdSize: Int, // nodeViewHolder: ActorRef, diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 9a8beaf0fa..f46f3baf63 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -6,7 +6,6 @@ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } @@ -15,11 +14,11 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ RollbackFailed, RollbackSucceed, SemanticallyFailedModification, - SemanticallySuccessfulModifier + SemanticallySuccessfulModifier, + SyntacticallyFailedModification } import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } -import encry.nvg.ModifiersValidator.ModifierForValidation -import encry.nvg.NetworkMessagesProcessor.IdsForRequest +import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation } import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage @@ -80,25 +79,26 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => intermediaryNetwork ! msg - case msg @ InvalidModifier(_) => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg - case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ FastSyncDone => - case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => + case msg @ BanPeer(_, _) => intermediaryNetwork ! msg + case msg @ InvalidNetworkBytes(_) => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg + case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ FastSyncDone => + case msg @ HeaderChainIsSynced => + case msg @ FullBlockChainIsSynced => //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => intermediaryNetwork ! msg networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index cb91e6d0d1..73d44dcfc2 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -14,9 +14,9 @@ import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } -import encry.network.DownloadedModifiersValidator.InvalidModifier +import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } +import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -32,7 +32,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case Left(error) => logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifier(modifierId) + context.parent ! InvalidNetworkBytes(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -44,13 +44,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! SyntacticallyInvalidPersistentModifier(modifierId) + context.parent ! SyntacticallyFailedModification(modifier, List.empty) } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) - context.parent ! InvalidModifier(modifierId) + context.parent ! SyntacticallyFailedModification(modifier, List.empty) } } } @@ -102,6 +102,8 @@ object ModifiersValidator { final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + final case class InvalidNetworkBytes(id: ModifierId) extends AnyVal + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } From 88e357395841d1591f198cdb7b356423e8d61961 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:00:49 +0300 Subject: [PATCH 034/119] made source in reqFromLocal optional --- src/main/scala/encry/network/Messages.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index faf0078240..10f881290e 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -8,7 +8,7 @@ object Messages { sealed trait MessageToNetwork object MessageToNetwork { - final case class RequestFromLocal(source: InetSocketAddress, + final case class RequestFromLocal(source: Option[InetSocketAddress], modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId]) extends MessageToNetwork final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork From db4d2ff2848b568a1a175227bf9b80c6e26713b3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:11:46 +0300 Subject: [PATCH 035/119] history reader implemented added sync info scheduler removed response from local message added nvg start --- src/main/scala/encry/Starter.scala | 21 +++++++---- .../scala/encry/network/DeliveryManager.scala | 3 +- .../scala/encry/nvg/IntermediaryNVH.scala | 37 +++++++++---------- .../scala/encry/nvg/ModifiersValidator.scala | 6 +-- .../encry/nvg/NetworkMessagesProcessor.scala | 17 +++++++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 24 ++++++------ .../encry/view/history/HistoryReader.scala | 35 ++++++++++-------- 7 files changed, 80 insertions(+), 63 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 25b5fd71e7..90980f3225 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -20,6 +20,7 @@ import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining import encry.network.{NetworkRouter, NodeViewSynchronizer} +import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender import encry.utils.{Mnemonic, NetworkTimeProvider} @@ -415,12 +416,12 @@ class Starter(settings: EncryAppSettings, .props(newSettings, timeProvider, miner, influxRef) .withDispatcher("mempool-dispatcher") ) - val nodeViewHolder: ActorRef = context.system.actorOf( - NodeViewHolder - .props(memoryPool, influxRef, dataHolderForApi, newSettings) - .withDispatcher("nvh-dispatcher"), - "nodeViewHolder" - ) +// val nodeViewHolder: ActorRef = context.system.actorOf( +// NodeViewHolder +// .props(memoryPool, influxRef, dataHolderForApi, newSettings) +// .withDispatcher("nvh-dispatcher"), +// "nodeViewHolder" +// ) if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) @@ -431,13 +432,19 @@ class Starter(settings: EncryAppSettings, // "nodeViewSynchronizer" // ) - context.system.actorOf( + val networkRouter = context.system.actorOf( NetworkRouter .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), "nodeViewSynchronizer" ) + val nvhRouter = context.system.actorOf( + IntermediaryNVH.props(settings, networkRouter, timeProvider, influxRef) + ) + + + if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 3fcd742e84..36774ef7ea 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -20,14 +20,13 @@ import scala.collection.{IndexedSeq, mutable} import scala.util.Random import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ModifiersForValidating} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.PeersKeeper.ConnectionStatusMessages.ConnectionStopped import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.nvg.NodeViewHolder.DownloadRequest import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f46f3baf63..4806d992e6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,25 +1,18 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef } +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ - OtherNodeSyncingStatus, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} -import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } -import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -33,7 +26,6 @@ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, - ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } @@ -76,17 +68,17 @@ class IntermediaryNVH( logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg - case UpdateHistoryReader(newReader: HistoryReader) => + case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader - networkMessagesProcessor ! newReader + networkMessagesProcessor ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg - case msg @ InvalidNetworkBytes(_) => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg + case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg + case msg @ SendSyncInfo(_) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => @@ -106,4 +98,11 @@ class IntermediaryNVH( } } -object IntermediaryNVH {} +object IntermediaryNVH { + def props( + settings: EncryAppSettings, + intermediaryNetwork: ActorRef, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef)) +} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 73d44dcfc2..a362d10c1b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -32,7 +32,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case Left(error) => logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidNetworkBytes(modifierId) + context.parent ! InvalidModifierBytes(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -102,7 +102,7 @@ object ModifiersValidator { final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal - final case class InvalidNetworkBytes(id: ModifierId) extends AnyVal + final case class InvalidModifierBytes(id: ModifierId) extends AnyVal def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 767cdbb013..9e33e7513e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -5,11 +5,11 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } -import encry.nvg.NodeViewHolder.DownloadRequest +import encry.nvg.NodeViewHolder.{ UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader @@ -25,6 +25,8 @@ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.concurrent.duration._ + class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { import context.dispatcher @@ -33,6 +35,11 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty + context.system.scheduler.schedule(5.seconds, settings.network.syncInterval) { + logger.debug("Scheduler once for SendLocalSyncInfo triggered") + context.parent ! SendSyncInfo(historyReader.syncInfo) + } + override def receive(): Receive = workingCycle( context.system.scheduler @@ -41,6 +48,8 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { + case UpdateHistoryReader(newReader: HistoryReader) => + historyReader = newReader case SemanticallySuccessfulModifier(mod) => //todo possible way to call CheckPayloadsToDownload mod match { @@ -86,7 +95,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St if (ids.nonEmpty && (invData._1 == Header.modifierTypeId || (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) - sender() ! RequestFromLocal(remote, invData._1, ids.toList) + sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => @@ -114,7 +123,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) context.parent ! DownloadRequest(Payload.modifierTypeId, newIds.toList) + if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) context.become(workingCycle(nextCheckModsScheduler.some)) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 60bfe7705c..22e9e971d9 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,18 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -23,7 +24,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -31,14 +32,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -158,8 +159,7 @@ class NodeViewHolder( updatedState.getOrElse(nodeView.state), updatedVault.getOrElse(nodeView.wallet) ) - if (updatedHistory.nonEmpty) context.parent ! ChangedHistory(newNodeView.history) - if (updatedState.nonEmpty) context.parent ! ChangedState(newNodeView.state) + if (updatedHistory.nonEmpty) context.parent ! UpdateHistoryReader(HistoryReader(newNodeView.history)) nodeView = newNodeView } @@ -171,7 +171,7 @@ class NodeViewHolder( s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) - context.parent ! DownloadRequest(tid, List(id)) + context.parent ! RequestFromLocal(none, tid, List(id)) else logger.info( s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 3f0640ef69..e4f652d869 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,6 +1,6 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -24,26 +24,29 @@ trait HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + def syncInfo: SyncInfo } object HistoryReader { def empty: HistoryReader = new HistoryReader { - def isModifierDefined(id: ModifierId): Boolean = true - def getBestHeaderHeight = 0 + def isModifierDefined(id: ModifierId): Boolean = false + def getBestHeaderHeight: Int = -1 def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty - var isFullChainSynced: Boolean = true - def compare(si: SyncInfo): HistoryComparisonResult = Older - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None - def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + var isFullChainSynced: Boolean = false + def compare(si: SyncInfo): HistoryComparisonResult = Older + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + def syncInfo: SyncInfo = SyncInfo(Seq.empty) } - def apply(): HistoryReader = new HistoryReader { - def isModifierDefined(id: ModifierId): Boolean = true - def getBestHeaderHeight = 1 - def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty - def compare(si: SyncInfo): HistoryComparisonResult = Older - var isFullChainSynced: Boolean = true - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None - def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + def apply(history: History): HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) + def getBestHeaderHeight: Int = history.getBestHeaderHeight + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) + def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) + var isFullChainSynced: Boolean = history.isFullChainSynced + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) + def syncInfo: SyncInfo = history.syncInfo } -} \ No newline at end of file +} From eeaf345f0878980434eea42f13c5d1c91ed130ea Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:14:17 +0300 Subject: [PATCH 036/119] remove redundant dependencies --- src/main/scala/encry/network/DeliveryManager.scala | 1 - .../scala/encry/network/DownloadedModifiersValidator.scala | 2 +- src/main/scala/encry/network/NodeViewSynchronizer.scala | 1 - src/main/scala/encry/view/mempool/MemoryPool.scala | 3 ++- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 36774ef7ea..dd7f913792 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -540,7 +540,6 @@ object DeliveryManager { case ConnectionStopped(_) => 1 - case InvalidModifier(_) => 2 // case DataFromPeer(msg: ModifiersNetworkMessage, _) => // msg match { diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 156bdc9009..8732b1d7eb 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -7,7 +7,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.modifiers.history.HeaderUtils import encry.network.BlackList.BanReason._ -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ ModifiersForValidating} import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 8dc068636f..ffa7d6c4c5 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -10,7 +10,6 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ import encry.local.miner.Miner.{DisableMining, ClIMiner, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 6cd2e3ccf3..fd35c9e567 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -7,7 +7,8 @@ import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider From 65682c26ac1da1c5d9e2df47d243c0a1c8647b56 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:26:21 +0300 Subject: [PATCH 037/119] add predicates to peers getting --- src/main/scala/encry/network/DM.scala | 4 +- .../scala/encry/network/MessageBuilder.scala | 40 ++++++++++++++++++- src/main/scala/encry/network/PK.scala | 7 +++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 60e775f32e..fda98e919a 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -13,7 +13,7 @@ import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} - +import cats.syntax.option._ import scala.collection.mutable case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { @@ -35,7 +35,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging AwaitingRequest(peer, modTypeId, modId, 1) ) case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => - if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer, modTypeId, List(modId)) + if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) case AwaitingRequest(peer, _, modId, _) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 952e95fcb9..46537536a0 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -6,8 +6,10 @@ import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.{Equal, Older, Younger} +import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} import encry.network.Messages.MessageToNetwork import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer @@ -26,7 +28,7 @@ case class MessageBuilder(msg: MessageToNetwork, implicit val timeout: Timeout = Timeout(10 seconds) override def receive: Receive = { - case RequestFromLocal(peer, modTypeId, modsIds) => + case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => modsIds.foreach { modId => @@ -39,6 +41,19 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + case RequestFromLocal(None, modTypeId, modsIds) => + Try { + (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => + modsIds.foreach { modId => + for { + isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] + } yield if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } + } + } case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! syncInfo) @@ -61,6 +76,27 @@ object MessageBuilder { case object GetPeers case class GetPeerInfo(peerIp: InetSocketAddress) + trait GetPeerByPredicate { + def predicate: PeerInfo => Boolean + def ||(that: GetPeerByPredicate): GetPeerByPredicate = new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = info => this.predicate(info) || that.predicate(info) + } + def &&(that: GetPeerByPredicate): GetPeerByPredicate =new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = info => this.predicate(info) && that.predicate(info) + } + } + final case class GetPeerWithEqualHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Equal + } + + final case class GetPeerWithOlderHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Older + } + + final case class GetPeerWithYoungerHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger + } + def props(msg: MessageToNetwork, peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index c8608160d0..3c892e5b0f 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -5,7 +5,7 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.{BanReason, BanTime, BanType} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} @@ -107,6 +107,11 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } + case predicate: GetPeerByPredicate => connectedPeers.getAll.find { + case (_, info) => predicate.predicate(info) + }.map { + case (_, info) => sender() ! info.connectedPeer.handlerRef + } case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { case (_, info) => sender() ! info.connectedPeer From 7c49193b802991af023382c598da5036bb1374c7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:28:23 +0300 Subject: [PATCH 038/119] improved rolled back txs processing --- src/main/scala/encry/nvg/NodeViewHolder.scala | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 22e9e971d9..3c01b585a6 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -24,7 +24,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -32,14 +32,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -148,7 +148,6 @@ class NodeViewHolder( } else () } - //todo replace outgoing message with history reader def updateNodeView( updatedHistory: Option[History] = none, updatedState: Option[UtxoState] = none, @@ -348,7 +347,7 @@ class NodeViewHolder( updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) - sendUpdatedInfoToMemoryPool(progressInfo.toRemove) + sendUpdatedInfoToMemoryPool(progressInfo.toRemove, progressInfo.toApply) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) @@ -373,12 +372,16 @@ class NodeViewHolder( } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { - val rolledBackTxs: IndexedSeq[Transaction] = toRemove + def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier], toApply: Seq[PersistentModifier]): Unit = { + val toRemoveTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq - //todo compare with toApply - if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) + val toApplyTxs: Vector[String] = toApply + .flatMap(extractTransactions) + .toVector + .map(_.encodedId) + val resultedTxs: IndexedSeq[Transaction] = toRemoveTxs.filterNot(tx => toApplyTxs.contains(tx.encodedId)) + if (resultedTxs.nonEmpty) context.parent ! RolledBackTransactions(resultedTxs) } def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { From 8876435484fd4e0202a6dc0c3fceeecad6b22b79 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:31:58 +0300 Subject: [PATCH 039/119] comment some tests --- .../DeliveryManagerRequestModifiesSpec.scala | 638 +++++++++--------- .../DownloadedModifiersValidatorTests.scala | 2 +- 2 files changed, 320 insertions(+), 320 deletions(-) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 067b867ecd..8a12695db0 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,319 +1,319 @@ -package encry.network.DeliveryManagerTests - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.TestNetSettings -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.collection.mutable.WrappedArray - -class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) - } - - "RequestModifies" should { - "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) - headersIds.foreach(id => - deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => - block.id.sameElements(id)).get.payload.id)) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .size == blocks.size) - deliveryManager.stop() - } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) - - deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers while block chain is not synced from Younger nodes" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection = - Map( - address2 -> (cp2, Younger, InitialPriority), - address3 -> (cp3, Fork, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - handler2.expectNoMsg() - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) - ) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) - assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) - deliveryManager.stop() - } - } -} \ No newline at end of file +//package encry.network.DeliveryManagerTests +// +//import java.net.InetSocketAddress +// +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.TestNetSettings +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.Messages.MessageToNetwork.RequestFromLocal +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +//import org.encryfoundation.common.network.SyncInfo +//import org.encryfoundation.common.utils.Algos +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +// +//import scala.collection.mutable.WrappedArray +// +//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) +// } +// +// "RequestModifies" should { +// "handle uniq modifiers from RequestFromLocal message correctly" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "not handle repeating modifiers from RequestFromLocal message" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery Manager should handle received modifier which were requested correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery manager should not handle repeating modifiers" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "handle priority request for payload correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) +// headersIds.foreach(id => +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => +// block.id.sameElements(id)).get.payload.id)) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .size == blocks.size) +// deliveryManager.stop() +// } +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) +// +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers while block chain is not synced from Younger nodes" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection = +// Map( +// address2 -> (cp2, Younger, InitialPriority), +// address3 -> (cp3, Fork, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// handler2.expectNoMsg() +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask transactions while block chain is not synced" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not ask transaction while node is not mining" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not re-ask modifiers which already have been received" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) +// ) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) +// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala index 2b8ffd799c..0b05c5bfb7 100644 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala @@ -6,7 +6,7 @@ import akka.actor.ActorSystem import akka.testkit.{ TestActorRef, TestProbe } import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +//import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, UpdatedHistory } import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } import encry.network.PeersKeeper.BanPeer From 88ef4dd05d8144823a85df241274979e3c5af56a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:48:11 +0300 Subject: [PATCH 040/119] messages position changed --- .../encry/api/http/DataHolderForApi.scala | 5 ++- src/main/scala/encry/local/miner/Miner.scala | 6 ++-- src/main/scala/encry/network/DM.scala | 3 +- .../encry/network/NodeViewSynchronizer.scala | 24 +++----------- .../scala/encry/nvg/IntermediaryNVH.scala | 9 +++++- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 4 +-- src/main/scala/encry/nvg/NodeViewHolder.scala | 31 +++++++++++++++++-- .../encry/view/fast/sync/SnapshotHolder.scala | 31 +++++++++---------- .../scala/encry/view/mempool/MemoryPool.scala | 4 +-- 10 files changed, 72 insertions(+), 47 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 7faaf529c0..935b592d8d 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -1,6 +1,7 @@ package encry.api.http import java.net.{InetAddress, InetSocketAddress} + import akka.actor.{Actor, ActorRef, Props, Stash} import akka.pattern._ import akka.util.Timeout @@ -20,9 +21,10 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI +import encry.nvg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} -import encry.view.NodeViewHolder.ReceivableMessages.{ GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet @@ -31,6 +33,7 @@ import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.concurrent.Future class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 9bc8bde878..660cb5324e 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -2,6 +2,7 @@ package encry.local.miner import java.text.SimpleDateFormat import java.util.Date + import akka.actor.{Actor, ActorRef, Props} import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging @@ -13,13 +14,13 @@ import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -34,6 +35,7 @@ import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} import org.encryfoundation.common.utils.constants.TestNetConstants + import scala.collection._ import scala.concurrent.duration._ diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index fda98e919a..965e62d027 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -8,12 +8,13 @@ import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork -import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} + import scala.collection.mutable case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index ffa7d6c4c5..0875248abc 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -2,13 +2,14 @@ package encry.network import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import akka.util.Timeout import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{DisableMining, ClIMiner, StartMining} +import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -29,11 +30,14 @@ import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, Tr import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} + import scala.util.Try //class NodeViewSynchronizer(influxRef: Option[ActorRef], @@ -228,32 +232,14 @@ object NodeViewSynchronizer { final case class RemovePeerFromBlackList(address: InetSocketAddress) extends CLIPeer - trait NodeViewHolderEvent - - trait NodeViewChange extends NodeViewHolderEvent - case class ChangedHistory(reader: History) extends NodeViewChange final case class UpdatedHistory(history: History) extends AnyVal case class ChangedState(reader: UtxoState) extends NodeViewChange - case class RollbackFailed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent - - case class RollbackSucceed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent - trait ModificationOutcome extends NodeViewHolderEvent - case class SyntacticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) - extends ModificationOutcome - - case class SemanticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) - extends ModificationOutcome - - case class SuccessfulTransaction(transaction: Transaction) extends ModificationOutcome - - case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome - } // def props(influxRef: Option[ActorRef], diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 4806d992e6..aa23f0af5d 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -12,7 +12,14 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandlin import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index a362d10c1b..e3990ec576 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -14,9 +14,9 @@ import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } -import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 9e33e7513e..17d755f257 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -8,8 +8,8 @@ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } -import encry.nvg.NodeViewHolder.{ UpdateHistoryReader } +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 3c01b585a6..ff11c3536b 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -14,7 +14,16 @@ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation } +import encry.nvg.NodeViewHolder.{ + NodeView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader, + UpdateInformation +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -31,7 +40,7 @@ import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos @@ -502,6 +511,24 @@ object NodeViewHolder { final case class LocallyGeneratedModifier(pmod: PersistentModifier) extends AnyVal } + trait NodeViewHolderEvent + + trait NodeViewChange extends NodeViewHolderEvent + + case class RollbackFailed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent + + case class RollbackSucceed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent + + case class SyntacticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) + extends ModificationOutcome + + case class SemanticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) + extends ModificationOutcome + + case class SuccessfulTransaction(transaction: Transaction) extends ModificationOutcome + + case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome + final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId] diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index 35e3ef98fc..c922650269 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -2,29 +2,28 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage -import akka.actor.{Actor, ActorRef, Cancellable, Props} -import cats.syntax.either._ +import akka.actor.{ Actor, ActorRef, Cancellable, Props } import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ChangedHistory, SemanticallySuccessfulModifier} -import encry.network.PeersKeeper.{BanPeer, SendToNetwork} -import encry.network.{Broadcast, PeerConnectionHandler} +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory +import encry.network.PeersKeeper.SendToNetwork +import encry.network.{ Broadcast, PeerConnectionHandler } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings -import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue } +import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ ChunkId, ManifestId } import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.History import encry.view.state.UtxoState -import encry.view.state.avlTree.{Node, NodeSerilalizer} +import encry.view.state.avlTree.{ Node, NodeSerilalizer } import encry.view.wallet.EncryWallet import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos import supertagged.TaggedType + import scala.util.Try class SnapshotHolder(settings: EncryAppSettings, @@ -278,21 +277,21 @@ class SnapshotHolder(settings: EncryAppSettings, if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => snapshotProcessor.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") - //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) + //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } case RequestManifestMessage(manifest) => logger.debug(s"Got request for manifest with ${Algos.encode(manifest)}") case RequestChunkMessage(chunkId) - //if requestsProcessor.canProcessRequest(remote) - => + //if requestsProcessor.canProcessRequest(remote) + => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) - //remote.handlerRef ! networkMessage + //remote.handlerRef ! networkMessage } - //requestsProcessor = requestsProcessor.processRequest(remote) + //requestsProcessor = requestsProcessor.processRequest(remote) case RequestChunkMessage(_) => case _ => } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index fd35c9e567..9635e325aa 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -8,8 +8,8 @@ import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews @@ -79,7 +79,7 @@ class MemoryPool(settings: EncryAppSettings, case CompareViews(peer, _, transactions) => val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) if (notYetRequestedTransactions.nonEmpty) { - sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) + //sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + From a5dfdef02383b94e8b6ebc4f733b6d5d25f36aca Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:48:37 +0300 Subject: [PATCH 041/119] fix --- src/main/scala/encry/Starter.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 90980f3225..cc74534a77 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -440,11 +440,9 @@ class Starter(settings: EncryAppSettings, ) val nvhRouter = context.system.actorOf( - IntermediaryNVH.props(settings, networkRouter, timeProvider, influxRef) + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) ) - - if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system From 5c1e871477dab9ffd37fd8860a259d0de24a032c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:57:31 +0300 Subject: [PATCH 042/119] fix message builder starting --- src/main/scala/encry/Starter.scala | 2 +- .../scala/encry/network/MessageBuilder.scala | 5 +++++ .../scala/encry/network/NetworkRouter.scala | 5 ++++- .../encry/view/wallet/AccountManager.scala | 20 +++++++++---------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index cc74534a77..6afb9ca961 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -436,7 +436,7 @@ class Starter(settings: EncryAppSettings, NetworkRouter .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), - "nodeViewSynchronizer" + "networkRouter" ) val nvhRouter = context.system.actorOf( diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 46537536a0..171ac31988 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -41,6 +41,7 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + context.stop(self) case RequestFromLocal(None, modTypeId, modsIds) => Try { (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => @@ -54,20 +55,24 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! syncInfo) } + context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) } } + context.stop(self) case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) } + context.stop(self) } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 2cd0ed37cd..8c19347344 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -19,7 +19,9 @@ import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConn import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.duration._ +import scala.util.Random class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { @@ -64,7 +66,8 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), "peersKeeper") + case msg: MessageToNetwork => + context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") } def peersLogic: Receive = { diff --git a/src/main/scala/encry/view/wallet/AccountManager.scala b/src/main/scala/encry/view/wallet/AccountManager.scala index a27b5d8391..1c5ec85582 100644 --- a/src/main/scala/encry/view/wallet/AccountManager.scala +++ b/src/main/scala/encry/view/wallet/AccountManager.scala @@ -65,16 +65,16 @@ case class AccountManager private(store: Store, password: String, mandatoryAccou object AccountManager { def init(mnemonicKey: String, pass: String, settings: EncryAppSettings): Unit = { - val keysTmpDir: File = new File(s"${settings.directory}/keysTmp") - val keysDir: File = new File(s"${settings.directory}/keys") - keysDir.mkdirs() - keysTmpDir.mkdirs() - val accountManagerStore: LSMStore = new LSMStore(keysDir, keepVersions = 0, keySize = 34) - val accountTmpManagerStore: LSMStore = new LSMStore(keysTmpDir, keepVersions = 0, keySize = 34) - val account = AccountManager.apply(accountManagerStore, pass, mnemonicKey, 0.toByte) - val tmpAccount = AccountManager.apply(accountTmpManagerStore, pass, mnemonicKey, 0.toByte) - account.store.close() - tmpAccount.store.close() + val keysTmpDir: File = new File(s"${settings.directory}/keysTmp") + val keysDir: File = new File(s"${settings.directory}/keys") + keysDir.mkdirs() + keysTmpDir.mkdirs() + val accountManagerStore: LSMStore = new LSMStore(keysDir, keepVersions = 0, keySize = 34) + val accountTmpManagerStore: LSMStore = new LSMStore(keysTmpDir, keepVersions = 0, keySize = 34) + val account = AccountManager.apply(accountManagerStore, pass, mnemonicKey, 0.toByte) + val tmpAccount = AccountManager.apply(accountTmpManagerStore, pass, mnemonicKey, 0.toByte) + account.store.close() + tmpAccount.store.close() } val AccountPrefix: Byte = 0x05 From d87bd12fc3f24a5cb9df218f260119a185243eac Mon Sep 17 00:00:00 2001 From: Lior Date: Thu, 5 Mar 2020 17:05:58 +0300 Subject: [PATCH 043/119] mempool changed --- .../view/mempool/IntermediaryMempool.scala | 57 ++++++++++ .../scala/encry/view/mempool/MemoryPool.scala | 88 ++++++--------- .../view/mempool/MemoryPoolProcessor.scala | 105 ++++++++++++++++++ .../view/mempool/TransactionsValidator.scala | 56 ++++++++++ 4 files changed, 250 insertions(+), 56 deletions(-) create mode 100644 src/main/scala/encry/view/mempool/IntermediaryMempool.scala create mode 100644 src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala create mode 100644 src/main/scala/encry/view/mempool/TransactionsValidator.scala diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala new file mode 100644 index 0000000000..1cc1da0dfa --- /dev/null +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -0,0 +1,57 @@ +package encry.view.mempool + +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.ValidatedModifierFromNetwork +import encry.utils.NetworkTimeProvider +import encry.view.mempool.IntermediaryMempool.IsChainSynced +import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} +import encry.view.mempool.MemoryPoolProcessor.RequestedModifiersForRemote +import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction + +class IntermediaryMempool(settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + minerReference: ActorRef, + influxReference: Option[ActorRef]) + extends Actor + with StrictLogging { + + val memoryPool: ActorRef = + context.actorOf(MemoryPool.props(settings, networkTimeProvider, self, influxReference), name = "mempool") + val txValidator: ActorRef = + context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), + name = "Transaction-validator") + val mempoolProcessor: ActorRef = + context.actorOf(MemoryPoolProcessor.props(settings), name = "mempool-processor") + + override def receive(): Receive = { + case msg @ InvalidModifier(_) => // to nvsh + case msg @ BanPeer(_, _) => // to peersKeeper + case msg @ ValidatedModifierFromNetwork(_) => // to influx + case msg @ TransactionsForMiner(_) => minerReference ! msg// to miner + case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool + case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool + case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor + case msg @ RequestedModifiersForRemote(_, _) => // to network + case msg @ RequestFromLocal(_, _, _) => // to network + case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor + case msg @ IsChainSynced(_) => mempoolProcessor ! msg + } +} + +object IntermediaryMempool { + def props(settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + minerReference: ActorRef, + influxReference: Option[ActorRef]) = + Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference)) + + final case class TransactionsForValidating(tx: Transaction) + + final case class IsChainSynced(info: Boolean) +} diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 9635e325aa..e4e81be6d1 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -3,11 +3,9 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ -import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings @@ -17,28 +15,22 @@ import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransac import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq class MemoryPool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, + intermediaryMempool: ActorRef, influxReference: Option[ActorRef]) extends Actor with StrictLogging { import context.dispatcher var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, networkTimeProvider) - var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + var canProcessTransactions: Boolean = false override def preStart(): Unit = { logger.debug(s"Starting MemoryPool. Initializing all schedulers") - context.system.eventStream.subscribe(self, classOf[NewTransaction]) - context.system.scheduler.schedule( - settings.mempool.bloomFilterCleanupInterval, - settings.mempool.bloomFilterCleanupInterval, self, CleanupBloomFilter) context.system.scheduler.schedule( settings.mempool.cleanupInterval, settings.mempool.cleanupInterval, self, RemoveExpiredFromPool) @@ -67,7 +59,8 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"MemoryPool has its limit of processed transactions. " + s"Transit to 'disableTransactionsProcessor' state." + s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StopTransactionsValidation) + canProcessTransactions = false + context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + 1 @@ -76,14 +69,14 @@ class MemoryPool(settings: EncryAppSettings, context.become(continueProcessing(currentTransactionsNumber)) } - case CompareViews(peer, _, transactions) => - val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) - if (notYetRequestedTransactions.nonEmpty) { - //sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) - logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + - s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") - } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + - s" There are no not yet requested ids.") +// case CompareViews(peer, _, transactions) => +// val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) +// if (notYetRequestedTransactions.nonEmpty) { +// sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) +// logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + +// s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") +// } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + +// s" There are no not yet requested ids.") case RolledBackTransactions(transactions) => val (newMemoryPool: MemoryPoolStorage, validatedTransactions: Seq[Transaction]) = @@ -95,7 +88,8 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"MemoryPool has its limit of processed transactions. " + s"Transit to 'disableTransactionsProcessor' state." + s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StopTransactionsValidation) + canProcessTransactions = false + context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + validatedTransactions.size @@ -111,20 +105,19 @@ class MemoryPool(settings: EncryAppSettings, s"Transit to a transactionsProcessor state.") if (state == NotProcessingNewTransactions) Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) + canProcessTransactions = true + context.parent ! TransactionProcessing(canProcessTransactions) context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) case SemanticallySuccessfulModifier(_) => logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + s"while $state. Do nothing in this case.") - case CleanupBloomFilter => - bloomFilterForTransactionsIds = initBloomFilter - case SendTransactionsToMiner => val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = memoryPool.getTransactionsForMiner memoryPool = newMemoryPool - minerReference ! TransactionsForMiner(transactionsForMiner) + intermediaryMempool ! TransactionsForMiner(transactionsForMiner) logger.debug(s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}") @@ -132,29 +125,17 @@ class MemoryPool(settings: EncryAppSettings, memoryPool = memoryPool.filter(memoryPool.isExpired) logger.debug(s"MemoryPool got RemoveExpiredFromPool message. After cleaning pool size is: ${memoryPool.size}.") - case RequestModifiersForTransactions(remote, ids) => - val modifiersIds: Seq[Transaction] = ids - .map(Algos.encode) - .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } - .flatten - sender() ! RequestedModifiersForRemote(remote, modifiersIds) - logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + - s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") +// case RequestModifiersForTransactions(remote, ids) => +// val modifiersIds: Seq[Transaction] = ids +// .map(Algos.encode) +// .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } +// .flatten +// sender() ! RequestedModifiersForRemote(remote, modifiersIds) +// logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + +// s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") case message => logger.debug(s"MemoryPool got unhandled message $message.") } - - def initBloomFilter: BloomFilter[String] = BloomFilter.create( - Funnels.stringFunnel(Charsets.UTF_8), - settings.mempool.bloomFilterCapacity, - settings.mempool.bloomFilterFailureProbability - ) - - def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { - case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => - bloomFilterForTransactionsIds.put(Algos.encode(id)) - id - } } object MemoryPool { @@ -165,15 +146,11 @@ object MemoryPool { final case class TransactionsForMiner(txs: Seq[Transaction]) extends AnyVal - final case class RequestModifiersForTransactions(peer: ConnectedPeer, txsIds: Seq[ModifierId]) - - final case class RequestedModifiersForRemote(peer: ConnectedPeer, txs: Seq[Transaction]) - case object SendTransactionsToMiner - case object RemoveExpiredFromPool + case class TransactionProcessing(info: Boolean) - case object CleanupBloomFilter + case object RemoveExpiredFromPool case object StopTransactionsValidation @@ -191,17 +168,16 @@ object MemoryPool { def props(settings: EncryAppSettings, ntp: NetworkTimeProvider, - minerRef: ActorRef, + intermediaryMempool: ActorRef, influx: Option[ActorRef]): Props = - Props(new MemoryPool(settings, ntp, minerRef, influx)) + Props(new MemoryPool(settings, ntp, intermediaryMempool, influx)) class MemoryPoolPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case RemoveExpiredFromPool | CleanupBloomFilter | SendTransactionsToMiner => 0 + case RemoveExpiredFromPool | SendTransactionsToMiner => 0 case NewTransaction(_) => 1 - case CompareViews(_, _, _) | RequestModifiersForTransactions(_, _) => 2 - case otherwise => 3 + case otherwise => 2 }) } \ No newline at end of file diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala new file mode 100644 index 0000000000..f9c3399ae4 --- /dev/null +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -0,0 +1,105 @@ +package encry.view.mempool + +import java.net.InetSocketAddress +import akka.actor.{ Actor, Props } +import com.google.common.base.Charsets +import com.google.common.hash.{ BloomFilter, Funnels } +import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings +import encry.view.mempool.IntermediaryMempool.IsChainSynced +import encry.view.mempool.MemoryPool.TransactionProcessing +import encry.view.mempool.MemoryPoolProcessor.{ CleanupBloomFilter, RequestedModifiersForRemote } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.collection.IndexedSeq + +class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + + import context.dispatcher + + var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + + var canProcessTransactions: Boolean = false + + var chainSynced: Boolean = false + + override def preStart(): Unit = + context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, + settings.mempool.bloomFilterCleanupInterval, + self, + CleanupBloomFilter) + + override def receive: Receive = { + + case TransactionProcessing(info) => canProcessTransactions = info + + case IsChainSynced(info) => chainSynced = info + + case CleanupBloomFilter => + bloomFilterForTransactionsIds = initBloomFilter + + case DataFromPeer(message, remote) => + message match { + + case RequestModifiersNetworkMessage((_, requestedIds)) => + val modifiersIds: Seq[Transaction] = requestedIds + .map(Algos.encode) + .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } + .flatten + context.parent ! RequestedModifiersForRemote(remote, modifiersIds) + logger.debug( + s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." + ) + + case InvNetworkMessage((_, txs)) => + val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) + if (notYetRequestedTransactions.nonEmpty) { + sender ! RequestFromLocal(Some(remote), Transaction.modifierTypeId, notYetRequestedTransactions.toList) + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" Not yet requested ids size is ${notYetRequestedTransactions.size}." + ) + } else + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" There are no not yet requested ids." + ) + + case InvNetworkMessage(invData) => + logger.debug( + s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions." + ) + + case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") + } + + } + def initBloomFilter: BloomFilter[String] = BloomFilter.create( + Funnels.stringFunnel(Charsets.UTF_8), + settings.mempool.bloomFilterCapacity, + settings.mempool.bloomFilterFailureProbability + ) + + def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { + case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => + bloomFilterForTransactionsIds.put(Algos.encode(id)) + id + } + +} + +object MemoryPoolProcessor { + + def props(settings: EncryAppSettings) = Props(new MemoryPoolProcessor(settings)) + + case object CleanupBloomFilter + + final case class RequestedModifiersForRemote(peer: InetSocketAddress, txs: Seq[Transaction]) + +} diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala new file mode 100644 index 0000000000..4268635170 --- /dev/null +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -0,0 +1,56 @@ +package encry.view.mempool + +import TransactionProto.TransactionProtoMessage +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer +import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider +import encry.view.mempool.MemoryPool.NewTransaction +import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.util.{Failure, Success, Try} + +class TransactionsValidator(settings: EncryAppSettings, + memPool: ActorRef, + networkTimeProvider: NetworkTimeProvider) + extends Actor + with StrictLogging { + + override def receive(): Receive = { + case ModifiersForValidating(remote, typeId, filteredModifiers) => + typeId match { + case Transaction.modifierTypeId => + filteredModifiers.foreach { + case (id, bytes) => + Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { + case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) + case Success(tx) => + logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") + context.parent ! BanPeer(remote.socketAddress, SyntacticallyInvalidTransaction) + context.parent ! InvalidModifier(id) + case Failure(ex) => + context.parent ! BanPeer(remote.socketAddress, CorruptedSerializedBytes) + context.parent ! InvalidModifier(id) + logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") + } + } + } + } + +} + +object TransactionsValidator { + + final case class ModifiersForValidating(remote: ConnectedPeer, + typeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]]) + + final case class InvalidModifier(ids: ModifierId) extends AnyVal + + def props(settings: EncryAppSettings, memPool: ActorRef, ntp: NetworkTimeProvider): Props = + Props(new TransactionsValidator(settings, memPool, ntp)) +} From d8b344d8fa253ef5923983b06ee73951cb57920d Mon Sep 17 00:00:00 2001 From: Lior Date: Thu, 5 Mar 2020 17:13:08 +0300 Subject: [PATCH 044/119] code cleanup --- .../encry/network/NodeViewSynchronizer.scala | 35 ++----------------- .../view/mempool/IntermediaryMempool.scala | 2 +- .../scala/encry/view/mempool/MemoryPool.scala | 4 +-- .../view/mempool/MemoryPoolProcessor.scala | 17 +++++---- 4 files changed, 16 insertions(+), 42 deletions(-) diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 0875248abc..dc569c7efa 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -1,45 +1,16 @@ package encry.network -import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress - -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} +import akka.actor.{ActorSystem, PoisonPill} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import akka.util.Timeout import com.typesafe.config.Config -import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer -import encry.network.PeersKeeper._ -import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic -import encry.settings.EncryAppSettings -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.Utils._ -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} -import encry.view.NodeViewErrors.ModifierApplyError +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.history.History import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState -import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} -import org.encryfoundation.common.modifiers.history._ -import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} -import org.encryfoundation.common.network.BasicMessagesRepo._ -import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} -import scala.concurrent.duration._ -import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} -import encry.view.fast.sync.SnapshotHolder -import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} - -import scala.util.Try - //class NodeViewSynchronizer(influxRef: Option[ActorRef], // nodeViewHolderRef: ActorRef, // settings: EncryAppSettings, diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 1cc1da0dfa..e798dbd897 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -27,7 +27,7 @@ class IntermediaryMempool(settings: EncryAppSettings, context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), name = "Transaction-validator") val mempoolProcessor: ActorRef = - context.actorOf(MemoryPoolProcessor.props(settings), name = "mempool-processor") + context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") override def receive(): Receive = { case msg @ InvalidModifier(_) => // to nvsh diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index e4e81be6d1..695ded5134 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -5,8 +5,6 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider @@ -128,7 +126,7 @@ class MemoryPool(settings: EncryAppSettings, // case RequestModifiersForTransactions(remote, ids) => // val modifiersIds: Seq[Transaction] = ids // .map(Algos.encode) -// .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } +// .collect { case if memorascyPool.contains(id) => memoryPool.get(id) } // .flatten // sender() ! RequestedModifiersForRemote(remote, modifiersIds) // logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index f9c3399ae4..0c919244b5 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,23 +1,26 @@ package encry.view.mempool import java.net.InetSocketAddress -import akka.actor.{ Actor, Props } + +import akka.actor.{Actor, Props} import com.google.common.base.Charsets -import com.google.common.hash.{ BloomFilter, Funnels } +import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{ CleanupBloomFilter, RequestedModifiersForRemote } +import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter, RequestedModifiersForRemote} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.IndexedSeq -class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { +class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { import context.dispatcher @@ -27,6 +30,8 @@ class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictL var chainSynced: Boolean = false + var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, ntp) + override def preStart(): Unit = context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, settings.mempool.bloomFilterCleanupInterval, @@ -96,7 +101,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictL object MemoryPoolProcessor { - def props(settings: EncryAppSettings) = Props(new MemoryPoolProcessor(settings)) + def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter From 8148c2d306afeaedc5a826cbc27c7be57fb8e5cd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 17:23:57 +0300 Subject: [PATCH 045/119] actor snapshot holder moved to the nvg --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 33 +++- .../scala/encry/nvg/IntermediaryNVH.scala | 7 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 12 +- .../SnapshotProcessorActor.scala} | 181 +++++++++--------- .../sync/SnapshotDownloadController.scala | 41 ++-- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- .../view/fast/sync/SnapshotProcessor.scala | 42 ++-- .../sync/SnapshotProcessorStorageAPI.scala | 3 +- .../encry/view/state/avlTree/AvlTree.scala | 4 +- 10 files changed, 186 insertions(+), 141 deletions(-) rename src/main/scala/encry/{view/fast/sync/SnapshotHolder.scala => nvg/SnapshotProcessorActor.scala} (70%) diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index fa63fc3e4e..5d9294009a 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,12 +7,12 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} +import encry.nvg.SnapshotProcessorActor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import org.encryfoundation.common.utils.TaggedTypes.Height diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index dc569c7efa..e3d4b4bbbb 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -1,16 +1,43 @@ package encry.network +import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress -import akka.actor.{ActorSystem, PoisonPill} + +import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.util.Timeout import com.typesafe.config.Config -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus._ +import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper._ +import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic +import encry.settings.EncryAppSettings +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.Utils._ +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} +import encry.view.NodeViewErrors.ModifierApplyError import encry.view.history.History import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.history._ +import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.network.BasicMessagesRepo._ +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.concurrent.duration._ +import encry.network.ModifiersToNetworkUtils._ +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} + +import scala.util.Try + //class NodeViewSynchronizer(influxRef: Option[ActorRef], // nodeViewHolderRef: ActorRef, // settings: EncryAppSettings, diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index aa23f0af5d..f088689cc5 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -20,15 +20,10 @@ import encry.nvg.NodeViewHolder.{ SyntacticallyFailedModification, UpdateHistoryReader } +import encry.nvg.SnapshotProcessorActor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider -import encry.view.fast.sync.SnapshotHolder.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index ff11c3536b..d29b40c6c0 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -24,14 +24,22 @@ import encry.nvg.NodeViewHolder.{ UpdateHistoryReader, UpdateInformation } +import encry.nvg.SnapshotProcessorActor.{ + FastSyncDone, + FastSyncFinished, + HeaderChainIsSynced, + RemoveRedundantManifestIds, + RequiredManifestHeightAndId, + SnapshotChunk, + TreeChunks +} +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/nvg/SnapshotProcessorActor.scala similarity index 70% rename from src/main/scala/encry/view/fast/sync/SnapshotHolder.scala rename to src/main/scala/encry/nvg/SnapshotProcessorActor.scala index c922650269..c19d543bc6 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessorActor.scala @@ -1,20 +1,28 @@ -package encry.view.fast.sync +package encry.nvg import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import akka.actor.{ Actor, ActorRef, Cancellable, Props } +import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{ + InvalidChunkMessage, + InvalidResponseManifestMessage, + InvalidStateAfterFastSync +} +import encry.network.Broadcast import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.SendToNetwork -import encry.network.{ Broadcast, PeerConnectionHandler } +import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.{ ChunkId, ManifestId } +import encry.nvg.SnapshotProcessorActor._ import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ ChunkId, ManifestId } -import encry.view.fast.sync.SnapshotHolder._ +import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } +import encry.view.fast.sync.{ RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotProcessor } import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree.{ Node, NodeSerilalizer } @@ -26,11 +34,12 @@ import supertagged.TaggedType import scala.util.Try -class SnapshotHolder(settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolder: ActorRef, - nodeViewSynchronizer: ActorRef) - extends Actor +class SnapshotProcessorActor( + settings: EncryAppSettings, + networkController: ActorRef, + nodeViewHolder: ActorRef, + nodeViewSynchronizer: ActorRef +) extends Actor with StrictLogging { import context.dispatcher @@ -93,43 +102,43 @@ class SnapshotHolder(settings: EncryAppSettings, logger.info( s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." ) -// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => -// (for { -// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) -// (controller, chunk) = controllerAndChunk -// validChunk <- snapshotProcessor.validateChunkId(chunk) -// processor = snapshotProcessor.updateCache(validChunk) -// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { -// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] -// case t => t.asLeft[SnapshotProcessor] -// } -// } yield (newProcessor, controller)) match { -// case Left(err: UnexpectedChunkMessage) => -// logger.info(s"Error has occurred ${err.error} with peer $remote") -// case Left(error) => -// logger.info(s"Error has occurred: $error") -// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) -// restartFastSync(history) -// case Right((processor, controller)) -// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => -// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) -// restartFastSync(history) -// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => -// processor.assembleUTXOState() match { -// case Right(state) => -// logger.info(s"Tree is valid on Snapshot holder!") -// processor.wallet.foreach { wallet: EncryWallet => -// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] -// } -// case _ => -// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) -// restartFastSync(history).asLeft[Unit] -// } -// case Right((processor, controller)) => -// snapshotDownloadController = controller -// snapshotProcessor = processor -// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks -// } + case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => + (for { + controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) + (controller, chunk) = controllerAndChunk + validChunk <- snapshotProcessor.validateChunkId(chunk) + processor = snapshotProcessor.updateCache(validChunk) + newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { + case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] + case t => t.asLeft[SnapshotProcessor] + } + } yield (newProcessor, controller)) match { + case Left(err: UnexpectedChunkMessage) => + logger.info(s"Error has occurred ${err.error} with peer $remote") + case Left(error) => + logger.info(s"Error has occurred: $error") + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) + restartFastSync(history) + case Right((processor, controller)) + if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) + restartFastSync(history) + case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => + processor.assembleUTXOState() match { + case Right(state) => + logger.info(s"Tree is valid on Snapshot holder!") + processor.wallet.foreach { wallet: EncryWallet => + (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] + } + case _ => + nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) + restartFastSync(history).asLeft[Unit] + } + case Right((processor, controller)) => + snapshotDownloadController = controller + snapshotProcessor = processor + if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + } case ResponseChunkMessage(_) => logger.info(s"Received chunk from unexpected peer ${remote}") @@ -149,9 +158,9 @@ class SnapshotHolder(settings: EncryAppSettings, case Right(controllerAndIds) => snapshotDownloadController = controllerAndIds._1 controllerAndIds._2.foreach { msg => - snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => - peer.handlerRef ! msg - } +// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +// peer.handlerRef ! msg +// } } context.become(fastSyncMod(history, timer).orElse(commonMessages)) } @@ -173,7 +182,7 @@ class SnapshotHolder(settings: EncryAppSettings, snapshotDownloadController.awaitedChunks.map { id => RequestChunkMessage(id.data) }.foreach { msg => - snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) + //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) } context.become(fastSyncMod(history, timer).orElse(commonMessages)) @@ -210,39 +219,39 @@ class SnapshotHolder(settings: EncryAppSettings, context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) case DataFromPeer(message, remote) => -// message match { -// case ResponseManifestMessage(manifest) => -// val isValidManifest: Boolean = -// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) -// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed -// if (isValidManifest && canBeProcessed) { -// (for { -// controller <- snapshotDownloadController.processManifest(manifest, remote, history) -// processor <- snapshotProcessor.initializeApplicableChunksCache( -// history, -// snapshotDownloadController.requiredManifestHeight -// ) -// } yield (controller, processor)) match { -// case Left(error) => -// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) -// case Right((controller, processor)) => -// logger.debug(s"Request manifest message successfully processed.") -// responseManifestTimeout.foreach(_.cancel()) -// snapshotDownloadController = controller -// snapshotProcessor = processor -// self ! RequestNextChunks -// logger.debug("Manifest processed successfully.") -// context.become(fastSyncMod(history, none)) -// } -// } else if (!isValidManifest) { -// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") -// nodeViewSynchronizer ! BanPeer( -// remote, -// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") -// ) -// } else logger.info(s"Doesn't need to process new manifest.") -// case _ => -// } + message match { + case ResponseManifestMessage(manifest) => + val isValidManifest: Boolean = + snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) + val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed + if (isValidManifest && canBeProcessed) { + (for { + controller <- snapshotDownloadController.processManifest(manifest, remote, history) + processor <- snapshotProcessor.initializeApplicableChunksCache( + history, + snapshotDownloadController.requiredManifestHeight + ) + } yield (controller, processor)) match { + case Left(error) => + nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) + case Right((controller, processor)) => + logger.debug(s"Request manifest message successfully processed.") + responseManifestTimeout.foreach(_.cancel()) + snapshotDownloadController = controller + snapshotProcessor = processor + self ! RequestNextChunks + logger.debug("Manifest processed successfully.") + context.become(fastSyncMod(history, none)) + } + } else if (!isValidManifest) { + logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") + nodeViewSynchronizer ! BanPeer( + remote, + InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") + ) + } else logger.info(s"Doesn't need to process new manifest.") + case _ => + } case msg @ RequiredManifestHeightAndId(_, _) => self ! msg @@ -316,7 +325,7 @@ class SnapshotHolder(settings: EncryAppSettings, context.system.scheduler.scheduleOnce(settings.snapshotSettings.responseTimeout)(self ! CheckDelivery).some } -object SnapshotHolder { +object SnapshotProcessorActor { case object RemoveRedundantManifestIds @@ -383,7 +392,7 @@ object SnapshotHolder { networkController: ActorRef, nodeViewHolderRef: ActorRef, nodeViewSynchronizer: ActorRef): Props = Props( - new SnapshotHolder(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) + new SnapshotProcessorActor(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) ) } diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index a9895bcfc2..588752bda5 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -1,46 +1,43 @@ package encry.view.fast.sync import java.io.File +import java.net.InetSocketAddress import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId -import encry.view.fast.sync.SnapshotHolder.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer} -import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.view.history.History import io.iohk.iodb.ByteArrayWrapper -import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, RequestChunkMessage } -import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.network.BasicMessagesRepo.RequestChunkMessage import org.encryfoundation.common.utils.Algos -import org.iq80.leveldb.{DB, Options} +import org.iq80.leveldb.{ DB, Options } -final case class SnapshotDownloadController(requiredManifestId: Array[Byte], - awaitedChunks: Set[ByteArrayWrapper], - settings: EncryAppSettings, - cp: Option[ConnectedPeer], - requiredManifestHeight: Int, - storage: DB, - batchesSize: Int, - nextGroupForRequestNumber: Int) - extends SnapshotDownloadControllerStorageAPI +final case class SnapshotDownloadController( + requiredManifestId: Array[Byte], + awaitedChunks: Set[ByteArrayWrapper], + settings: EncryAppSettings, + cp: Option[InetSocketAddress], + requiredManifestHeight: Int, + storage: DB, + batchesSize: Int, + nextGroupForRequestNumber: Int +) extends SnapshotDownloadControllerStorageAPI with StrictLogging with AutoCloseable { def processManifest( manifestProto: SnapshotManifestProtoMessage, - remote: ConnectedPeer, + remote: InetSocketAddress, history: History ): Either[SnapshotDownloadControllerException, SnapshotDownloadController] = { - logger.info(s"Got new manifest from ${remote.socketAddress}.") + logger.info(s"Got new manifest from $remote.") Either.fromTry(SnapshotManifestSerializer.fromProto(manifestProto)) match { case Left(error) => logger.info(s"Manifest was parsed with error ${error.getCause}.") @@ -74,9 +71,9 @@ final case class SnapshotDownloadController(requiredManifestId: Array[Byte], def processRequestedChunk( chunkMessage: SnapshotChunkMessage, - remote: ConnectedPeer + remote: InetSocketAddress ): Either[ChunkValidationError, (SnapshotDownloadController, SnapshotChunk)] = { - logger.debug(s"Got new chunk from ${remote.socketAddress}.") + logger.debug(s"Got new chunk from $remote.") Either.fromTry(SnapshotChunkSerializer.fromProto(chunkMessage)) match { case Left(error) => logger.info(s"Chunk was parsed with error ${error.getCause}.") @@ -123,7 +120,7 @@ final case class SnapshotDownloadController(requiredManifestId: Array[Byte], def canNewManifestBeProcessed: Boolean = cp.isEmpty - def canChunkBeProcessed(remote: ConnectedPeer): Boolean = cp.exists(_.socketAddress == remote.socketAddress) + def canChunkBeProcessed(remote: InetSocketAddress): Boolean = cp.contains(remote) def reInitFastSync: SnapshotDownloadController = try { diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index b01c7ae46c..b0ec686518 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,8 +1,8 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala index 3675c8dbce..07d25cc879 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala @@ -1,42 +1,50 @@ package encry.view.fast.sync import java.io.File + import akka.actor.ActorRef import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.{ + SnapshotChunk, + SnapshotChunkSerializer, + SnapshotManifest, + SnapshotManifestSerializer +} +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings -import encry.storage.{RootNodesStorage, VersionalStorage} -import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} +import encry.storage.{ RootNodesStorage, VersionalStorage } +import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } import encry.storage.iodb.versionalIODB.IODBWrapper -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer} import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree._ import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.wallet.EncryWallet -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import io.iohk.iodb.{ ByteArrayWrapper, LSMStore } import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} -import org.iq80.leveldb.{DB, Options} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.iq80.leveldb.{ DB, Options } import scorex.utils.Random -import scala.collection.immutable.{HashMap, HashSet} + +import scala.collection.immutable.{ HashMap, HashSet } import scala.language.postfixOps -import scala.util.{Failure, Success} +import scala.util.{ Failure, Success } -final case class SnapshotProcessor(settings: EncryAppSettings, - storage: VersionalStorage, - applicableChunks: HashSet[ByteArrayWrapper], - chunksCache: HashMap[ByteArrayWrapper, SnapshotChunk], - wallet: Option[EncryWallet]) - extends StrictLogging +final case class SnapshotProcessor( + settings: EncryAppSettings, + storage: VersionalStorage, + applicableChunks: HashSet[ByteArrayWrapper], + chunksCache: HashMap[ByteArrayWrapper, SnapshotChunk], + wallet: Option[EncryWallet] +) extends StrictLogging with SnapshotProcessorStorageAPI with AutoCloseable { @@ -181,7 +189,7 @@ final case class SnapshotProcessor(settings: EncryAppSettings, rootNode <- getRootNode height <- getHeight //todo: remove RootNodesStorage.emptyRootStorage - avlTree = new AvlTree[StorageKey, StorageValue](rootNode, storage, RootNodesStorage.emptyRootStorage) + avlTree = new AvlTree[StorageKey, StorageValue](rootNode, storage, RootNodesStorage.emptyRootStorage) } yield UtxoState(avlTree, height, settings.constants, influxRef) private def getHeight: Either[EmptyHeightKey, Height] = diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index 5ebbe5250f..c0f868cbb3 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,10 +3,11 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.SnapshotHolder.{ SnapshotManifest, SnapshotManifestSerializer } import org.encryfoundation.common.utils.Algos + import scala.util.Try trait SnapshotProcessorStorageAPI extends StrictLogging { diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 20c039bf6c..983cd6317a 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,10 +4,10 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.SnapshotChunk +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} -import encry.view.fast.sync.SnapshotHolder.SnapshotChunk -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree.Direction import encry.view.state.avlTree.AvlTree.Directions.{EMPTY, LEFT, RIGHT} From 548191bbd26fbe72e8310742c55d3e9bed54b431 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 18:10:32 +0300 Subject: [PATCH 046/119] add processing getPeers message on pk --- .../scala/encry/network/NetworkRouter.scala | 6 +- src/main/scala/encry/network/PK.scala | 52 ++++- ...hotDownloadControllerStorageAPITests.scala | 90 ++++---- .../sync/SnapshotDownloadControllerTest.scala | 212 +++++++++--------- 4 files changed, 205 insertions(+), 155 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 8c19347344..8d95e93c67 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -34,8 +34,8 @@ class NetworkRouter(settings: NetworkSettings, IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") - val deliveryManager = context.system.actorOf(DM.props(settings), "deliveryManager") + val peersKeeper = context.actorOf(PK.props(settings, blackListSettings), "peersKeeper") + val deliveryManager = context.actorOf(DM.props(settings), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress override def receive: Receive = bindingLogic orElse businessLogic orElse peersLogic orElse { @@ -67,7 +67,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => - context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") + context.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") } def peersLogic: Receive = { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 3c892e5b0f..6f0067b68e 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -4,13 +4,18 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} +import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} import scala.util.{Random, Try} @@ -33,7 +38,14 @@ class PK(networkSettings: NetworkSettings, var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - override def receive: Receive = banPeersLogic orElse { + override def preStart(): Unit = { + context.parent ! RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), self) + } + + override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { case RequestPeerForConnection if connectedPeers.size < networkSettings.maxConnections => def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + @@ -58,6 +70,8 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } + case OtherNodeSyncingStatus(remote, comparison, _) => + connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + @@ -118,6 +132,40 @@ class PK(networkSettings: NetworkSettings, } } + def networkMessagesProcessingLogic: Receive = { + case DataFromPeer(message, remote) => message match { + case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => + logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") + peers + .filterNot { p => + blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) + }.foreach { p => + logger.info(s"Found new peer: $p. Adding it to the available peers collection.") + peersForConnection = peersForConnection.updated(p, 0) + } + logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") + + case PeersNetworkMessage(_) => + logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + + s"so ignore this message and ban this peer.") + self ! BanPeer(remote, SentPeersMessageWithoutRequest) + + case GetPeersNetworkMessage => + def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = + Try { + if (remote.getAddress.isSiteLocalAddress) true + else add.getAddress.isSiteLocalAddress && add != remote + }.getOrElse(false) + + val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) + logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") + logger.info(s"Remote is side local: ${remote} : ${Try(remote.getAddress.isSiteLocalAddress)}") + connectedPeers.getAll.find(_._1 == remote).foreach { + case (_, info) => info.connectedPeer.handlerRef ! PeersNetworkMessage(peers) + } + } + } + def banPeersLogic: Receive = { case BanPeer(peer, reason) => logger.info(s"Banning peer: ${peer} for $reason.") @@ -129,6 +177,8 @@ class PK(networkSettings: NetworkSettings, blackList = blackList.banPeer(reason, peer.getAddress) } + def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || diff --git a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala index d07e2029f3..b1bc8ad5eb 100644 --- a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala +++ b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala @@ -1,45 +1,45 @@ -package encry.view.fast.sync - -import encry.settings.EncryAppSettings -import encry.storage.levelDb.versionalLevelDB.LevelDbFactory -import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId -import org.iq80.leveldb.{ DB, Options } -import org.scalatest.{ Matchers, WordSpecLike } -import scorex.utils.Random - -class SnapshotDownloadControllerStorageAPITests extends WordSpecLike with Matchers { - - val settingsR: EncryAppSettings = EncryAppSettings.read() - - def init: SnapshotDownloadControllerStorageAPI = new SnapshotDownloadControllerStorageAPI { - override val storage: DB = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) - override val settings: EncryAppSettings = settingsR - } - - "Inside SnapshotDownloadControllerStorageAPI class" should { - "insert many should insert all ids correctly / split for groups with correct size" in { - val api: SnapshotDownloadControllerStorageAPI = init - val randomIds: List[ChunkId] = (1 to 20001).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) - val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - val insertionsResult = api.insertMany(groups) - insertionsResult.isRight shouldBe true - } - "get next for request should return batch if such exists / remove returned batch" in { - val api: SnapshotDownloadControllerStorageAPI = init - val randomIds: List[ChunkId] = (1 to 5000).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) - val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - val _ = api.insertMany(groups) - val groupsL = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - (0 until groupsL.size).foreach { r => - val res = api.getNextForRequest(r) - api.getNextForRequest(r).isLeft shouldBe true - res.isRight shouldBe true - res.right.get.nonEmpty shouldBe true - res.right.get.head.sameElements(groupsL(r).head) shouldBe true - res.right.get.forall(j => groupsL(r).exists(_.sameElements(j))) shouldBe true - groupsL(r).forall(j => res.right.get.exists(_.sameElements(j))) shouldBe true - } - } - } -} +//package encry.view.fast.sync +// +//import encry.settings.EncryAppSettings +//import encry.storage.levelDb.versionalLevelDB.LevelDbFactory +//import encry.utils.FileHelper +//import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId +//import org.iq80.leveldb.{ DB, Options } +//import org.scalatest.{ Matchers, WordSpecLike } +//import scorex.utils.Random +// +//class SnapshotDownloadControllerStorageAPITests extends WordSpecLike with Matchers { +// +// val settingsR: EncryAppSettings = EncryAppSettings.read() +// +// def init: SnapshotDownloadControllerStorageAPI = new SnapshotDownloadControllerStorageAPI { +// override val storage: DB = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) +// override val settings: EncryAppSettings = settingsR +// } +// +// "Inside SnapshotDownloadControllerStorageAPI class" should { +// "insert many should insert all ids correctly / split for groups with correct size" in { +// val api: SnapshotDownloadControllerStorageAPI = init +// val randomIds: List[ChunkId] = (1 to 20001).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) +// val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// val insertionsResult = api.insertMany(groups) +// insertionsResult.isRight shouldBe true +// } +// "get next for request should return batch if such exists / remove returned batch" in { +// val api: SnapshotDownloadControllerStorageAPI = init +// val randomIds: List[ChunkId] = (1 to 5000).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) +// val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// val _ = api.insertMany(groups) +// val groupsL = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// (0 until groupsL.size).foreach { r => +// val res = api.getNextForRequest(r) +// api.getNextForRequest(r).isLeft shouldBe true +// res.isRight shouldBe true +// res.right.get.nonEmpty shouldBe true +// res.right.get.head.sameElements(groupsL(r).head) shouldBe true +// res.right.get.forall(j => groupsL(r).exists(_.sameElements(j))) shouldBe true +// groupsL(r).forall(j => res.right.get.exists(_.sameElements(j))) shouldBe true +// } +// } +// } +//} diff --git a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala index af09dbb389..4be8c49554 100644 --- a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala +++ b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala @@ -1,106 +1,106 @@ -package encry.view.fast.sync - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.TestProbe -import encry.modifiers.InstanceFactory -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.{EncryAppSettings, TestNetSettings} -import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} -import encry.view.fast.sync.SnapshotHolder.{SnapshotManifest, SnapshotManifestSerializer} -import org.encryfoundation.common.network.BasicMessagesRepo.Handshake -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} -import scorex.utils.Random - -class SnapshotDownloadControllerTest - extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - "Snapshot download controller" should { - "process new manifest message correctly" in { - val settingsWithRandomDir = EncryAppSettings - .read() - .copy( - directory = FileHelper.getRandomTempDir.toString - ) - val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) - val history = generateDummyHistory(settings) - val randomChunks = (0 to 20001).map(_ => ChunkId @@ Random.randomBytes()).toList - val randomManifest = SnapshotManifestSerializer.toProto( - SnapshotManifest( - ManifestId @@ Random.randomBytes(), - randomChunks - ) - ) - val address = new InetSocketAddress("0.0.0.0", 9000) - val peer: ConnectedPeer = ConnectedPeer( - address, - TestProbe().ref, - Incoming, - Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) - ) - val newController = snapshotDownloadController.processManifest( - randomManifest, - peer, - history - ) - - val requiredBatchesSize = - (randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod) + 1 - - newController.isRight shouldBe true - newController.right.get.batchesSize shouldBe requiredBatchesSize - } - "provide correct getNextBatchAndRemoveItFromController function" in { - val settingsWithRandomDir = EncryAppSettings - .read() - .copy( - directory = FileHelper.getRandomTempDir.toString - ) - val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) - val history = generateDummyHistory(settings) - val randomChunks = (1 to 20000).map(_ => ChunkId @@ Random.randomBytes()).toList - val randomManifest = SnapshotManifestSerializer.toProto( - SnapshotManifest( - ManifestId @@ Random.randomBytes(), - randomChunks - ) - ) - val address = new InetSocketAddress("0.0.0.0", 9000) - val peer: ConnectedPeer = ConnectedPeer( - address, - TestProbe().ref, - Incoming, - Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) - ) - val newController = snapshotDownloadController.processManifest( - randomManifest, - peer, - history - ) - val requiredBatchesSize = randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod - - val nextController = newController.right.get.getNextBatchAndRemoveItFromController - nextController.isRight shouldBe true - nextController.right.get._1.nextGroupForRequestNumber shouldBe 1 - nextController.right.get._1.batchesSize shouldBe requiredBatchesSize - 1 - - val nextController1 = (nextController.right.get._1.nextGroupForRequestNumber until requiredBatchesSize) - .foldLeft(nextController.right.get._1) { - case (controllerN, _) => - controllerN.getNextBatchAndRemoveItFromController.right.get._1 - } - - nextController1.batchesSize shouldBe 0 - - nextController1.getNextBatchAndRemoveItFromController.isLeft shouldBe true - - } - } -} +//package encry.view.fast.sync +// +//import java.net.InetSocketAddress +// +//import akka.actor.ActorSystem +//import akka.testkit.TestProbe +//import encry.modifiers.InstanceFactory +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import encry.utils.FileHelper +//import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} +//import encry.view.fast.sync.SnapshotHolder.{SnapshotManifest, SnapshotManifestSerializer} +//import org.encryfoundation.common.network.BasicMessagesRepo.Handshake +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +//import scorex.utils.Random +// +//class SnapshotDownloadControllerTest +// extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// "Snapshot download controller" should { +// "process new manifest message correctly" in { +// val settingsWithRandomDir = EncryAppSettings +// .read() +// .copy( +// directory = FileHelper.getRandomTempDir.toString +// ) +// val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) +// val history = generateDummyHistory(settings) +// val randomChunks = (0 to 20001).map(_ => ChunkId @@ Random.randomBytes()).toList +// val randomManifest = SnapshotManifestSerializer.toProto( +// SnapshotManifest( +// ManifestId @@ Random.randomBytes(), +// randomChunks +// ) +// ) +// val address = new InetSocketAddress("0.0.0.0", 9000) +// val peer: ConnectedPeer = ConnectedPeer( +// address, +// TestProbe().ref, +// Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) +// ) +// val newController = snapshotDownloadController.processManifest( +// randomManifest, +// peer, +// history +// ) +// +// val requiredBatchesSize = +// (randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod) + 1 +// +// newController.isRight shouldBe true +// newController.right.get.batchesSize shouldBe requiredBatchesSize +// } +// "provide correct getNextBatchAndRemoveItFromController function" in { +// val settingsWithRandomDir = EncryAppSettings +// .read() +// .copy( +// directory = FileHelper.getRandomTempDir.toString +// ) +// val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) +// val history = generateDummyHistory(settings) +// val randomChunks = (1 to 20000).map(_ => ChunkId @@ Random.randomBytes()).toList +// val randomManifest = SnapshotManifestSerializer.toProto( +// SnapshotManifest( +// ManifestId @@ Random.randomBytes(), +// randomChunks +// ) +// ) +// val address = new InetSocketAddress("0.0.0.0", 9000) +// val peer: ConnectedPeer = ConnectedPeer( +// address, +// TestProbe().ref, +// Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) +// ) +// val newController = snapshotDownloadController.processManifest( +// randomManifest, +// peer, +// history +// ) +// val requiredBatchesSize = randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod +// +// val nextController = newController.right.get.getNextBatchAndRemoveItFromController +// nextController.isRight shouldBe true +// nextController.right.get._1.nextGroupForRequestNumber shouldBe 1 +// nextController.right.get._1.batchesSize shouldBe requiredBatchesSize - 1 +// +// val nextController1 = (nextController.right.get._1.nextGroupForRequestNumber until requiredBatchesSize) +// .foldLeft(nextController.right.get._1) { +// case (controllerN, _) => +// controllerN.getNextBatchAndRemoveItFromController.right.get._1 +// } +// +// nextController1.batchesSize shouldBe 0 +// +// nextController1.getNextBatchAndRemoveItFromController.isLeft shouldBe true +// +// } +// } +//} From 87c0d195d5c0b89dff3b8c656ccff7a70ec9bf02 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 10:07:52 +0300 Subject: [PATCH 047/119] SyncInfo error was fixed --- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index d29b40c6c0..ba4c4c8c84 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -70,6 +70,8 @@ class NodeViewHolder( var nodeView: NodeView = restoreState().getOrElse(genesisState) + context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) + var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) From b2f4f8c5b9c69a75fe64f420e06a23ec1b766b0d Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 10:38:58 +0300 Subject: [PATCH 048/119] local commit --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 31 +++- src/main/scala/encry/nvg/NodeViewHolder.scala | 7 +- .../scala/encry/nvg/SnapshotDownloader.scala | 140 ++++++++++++++++++ .../encry/nvg/SnapshotIntermediary.scala | 12 ++ ...sorActor.scala => SnapshotProcessor.scala} | 88 ++++------- .../view/fast/sync/FastSyncExceptions.scala | 2 +- .../sync/RequestsPerPeriodProcessor.scala | 2 +- .../sync/SnapshotDownloadController.scala | 4 +- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- ...otProcessor.scala => SnapshotHolder.scala} | 50 +++---- .../sync/SnapshotProcessorStorageAPI.scala | 2 +- .../encry/view/state/avlTree/AvlTree.scala | 4 +- 13 files changed, 250 insertions(+), 96 deletions(-) create mode 100644 src/main/scala/encry/nvg/SnapshotDownloader.scala create mode 100644 src/main/scala/encry/nvg/SnapshotIntermediary.scala rename src/main/scala/encry/nvg/{SnapshotProcessorActor.scala => SnapshotProcessor.scala} (83%) rename src/main/scala/encry/view/fast/sync/{SnapshotProcessor.scala => SnapshotHolder.scala} (91%) diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index 5d9294009a..bcc7d55023 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,7 +7,7 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} -import encry.nvg.SnapshotProcessorActor +import encry.nvg.SnapshotProcessor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f088689cc5..92a7054b5a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -2,6 +2,7 @@ package encry.nvg import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } @@ -20,7 +21,7 @@ import encry.nvg.NodeViewHolder.{ SyntacticallyFailedModification, UpdateHistoryReader } -import encry.nvg.SnapshotProcessorActor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } +import encry.nvg.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -28,7 +29,11 @@ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, SyncInfoNetworkMessage } import org.encryfoundation.common.utils.Algos @@ -62,6 +67,22 @@ class IntermediaryNVH( .props(ModifiersValidator.props(nodeViewHolder, settings)), name = "Modifiers-validator-router" ) + val snapshotProcessor: Option[ActorRef] = + if (settings.constants.SnapshotCreationHeight <= settings.constants.MaxRollbackDepth || + (!settings.snapshotSettings.enableFastSynchronization && !settings.snapshotSettings.enableSnapshotCreation)) + none[ActorRef] + else { + intermediaryNetwork ! RegisterMessagesHandler( + Seq( + RequestManifestMessage.NetworkMessageTypeID -> "RequestManifest", + ResponseManifestMessage.NetworkMessageTypeID -> "ResponseManifestMessage", + RequestChunkMessage.NetworkMessageTypeID -> "RequestChunkMessage", + ResponseChunkMessage.NetworkMessageTypeID -> "ResponseChunkMessage" + ), + self + ) + context.actorOf(SnapshotProcessor.props(settings, nodeViewHolder)).some + } var historyReader: HistoryReader = HistoryReader.empty @@ -69,7 +90,13 @@ class IntermediaryNVH( case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) - case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: SyncInfoNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: InvNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: RequestModifiersNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: RequestManifestMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: ResponseManifestMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: RequestChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: ResponseChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index ba4c4c8c84..f62007ec6e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -24,7 +24,7 @@ import encry.nvg.NodeViewHolder.{ UpdateHistoryReader, UpdateInformation } -import encry.nvg.SnapshotProcessorActor.{ +import encry.nvg.SnapshotProcessor.{ FastSyncDone, FastSyncFinished, HeaderChainIsSynced, @@ -33,7 +33,7 @@ import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, TreeChunks } -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -70,10 +70,9 @@ class NodeViewHolder( var nodeView: NodeView = restoreState().getOrElse(genesisState) - context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) - var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) context.system.scheduler.schedule(1.seconds, 10.seconds) { diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/SnapshotDownloader.scala new file mode 100644 index 0000000000..408d1473b9 --- /dev/null +++ b/src/main/scala/encry/nvg/SnapshotDownloader.scala @@ -0,0 +1,140 @@ +package encry.nvg + +import akka.actor.{Actor, Cancellable} +import cats.syntax.option._ +import cats.syntax.either._ +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidStateAfterFastSync} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer +import encry.nvg.SnapshotProcessor.{CheckDelivery, FastSyncDone, FastSyncFinished, RequestNextChunks, RequiredManifestHeightAndId} +import encry.settings.EncryAppSettings +import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} +import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} +import encry.view.history.{History, HistoryReader} +import encry.view.wallet.EncryWallet +import org.encryfoundation.common.network.BasicMessagesRepo.{RequestChunkMessage, ResponseChunkMessage, ResponseManifestMessage} +import org.encryfoundation.common.utils.Algos + +class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { + + var snapshotHolder: SnapshotHolder = + SnapshotHolder.initialize( + settings, + if (settings.snapshotSettings.enableFastSynchronization) settings.storage.state + else settings.storage.snapshotHolder + ) + var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) + var historyReader: HistoryReader = HistoryReader.empty + + override def receive: Receive = ??? + + def fastSyncMod( + history: History, + responseTimeout: Option[Cancellable] + ): Receive = { + case DataFromPeer(message, remote) => + logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") + message match { + case ResponseManifestMessage(manifest) => + logger.info( + s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." + ) + case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => + (for { + controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) + (controller, chunk) = controllerAndChunk + validChunk <- snapshotHolder.validateChunkId(chunk) + processor = snapshotHolder.updateCache(validChunk) + newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { + case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] + case t => t.asLeft[SnapshotHolder] + } + } yield (newProcessor, controller)) match { + case Left(err: UnexpectedChunkMessage) => + logger.info(s"Error has occurred ${err.error} with peer $remote") + case Left(error) => + logger.info(s"Error has occurred: $error") + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) + restartFastSync(history) + case Right((processor, controller)) + if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) + restartFastSync(history) + case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => + processor.assembleUTXOState() match { + case Right(state) => + logger.info(s"Tree is valid on Snapshot holder!") + processor.wallet.foreach { wallet: EncryWallet => + (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] + } + case _ => + nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) + restartFastSync(history).asLeft[Unit] + } + case Right((processor, controller)) => + snapshotDownloadController = controller + snapshotHolder = processor + if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + } + + case ResponseChunkMessage(_) => + logger.info(s"Received chunk from unexpected peer ${remote}") + + case _ => + } + + case RequestNextChunks => + responseTimeout.foreach(_.cancel()) + (for { + controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController + _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") + } yield controllerAndIds) match { + case Left(err) => + logger.info(s"Error has occurred: ${err.error}") + throw new Exception(s"Error has occurred: ${err.error}") + case Right(controllerAndIds) => + snapshotDownloadController = controllerAndIds._1 + controllerAndIds._2.foreach { msg => + // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => + // peer.handlerRef ! msg + // } + } + context.become(fastSyncMod(history, timer).orElse(commonMessages)) + } + + case RequiredManifestHeightAndId(height, manifestId) => + logger.info( + s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + + s"New required manifest id is ${Algos.encode(manifestId)}." + ) + snapshotDownloadController = snapshotDownloadController.copy( + requiredManifestHeight = height, + requiredManifestId = manifestId + ) + restartFastSync(history) + self ! BroadcastManifestRequestMessage + context.become(awaitManifestMod(none, history).orElse(commonMessages)) + + case CheckDelivery => + snapshotDownloadController.awaitedChunks.map { id => + RequestChunkMessage(id.data) + }.foreach { msg => + //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) + } + context.become(fastSyncMod(history, timer).orElse(commonMessages)) + + case FastSyncDone => + if (settings.snapshotSettings.enableSnapshotCreation) { + logger.info(s"Snapshot holder context.become to snapshot processing") + snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) + snapshotDownloadController.storage.close() + context.system.scheduler + .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) + context.become(workMod(history).orElse(commonMessages)) + } else { + logger.info(s"Stop processing snapshots") + context.stop(self) + } + } +} diff --git a/src/main/scala/encry/nvg/SnapshotIntermediary.scala b/src/main/scala/encry/nvg/SnapshotIntermediary.scala new file mode 100644 index 0000000000..9e08c81b36 --- /dev/null +++ b/src/main/scala/encry/nvg/SnapshotIntermediary.scala @@ -0,0 +1,12 @@ +package encry.nvg + +import akka.actor.Actor + +class SnapshotIntermediary extends Actor { + + + + override def receive: Receive = { + case _ => + } +} diff --git a/src/main/scala/encry/nvg/SnapshotProcessorActor.scala b/src/main/scala/encry/nvg/SnapshotProcessor.scala similarity index 83% rename from src/main/scala/encry/nvg/SnapshotProcessorActor.scala rename to src/main/scala/encry/nvg/SnapshotProcessor.scala index c19d543bc6..bc52858361 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessorActor.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessor.scala @@ -2,30 +2,26 @@ package encry.nvg import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage -import akka.actor.{ Actor, ActorRef, Cancellable, Props } +import akka.actor.{Actor, ActorRef, Cancellable, Props} import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{ - InvalidChunkMessage, - InvalidResponseManifestMessage, - InvalidStateAfterFastSync -} +import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} import encry.network.Broadcast -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } +import encry.network.PeersKeeper.{BanPeer, SendToNetwork} import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.{ ChunkId, ManifestId } -import encry.nvg.SnapshotProcessorActor._ +import encry.nvg.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings -import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } -import encry.view.fast.sync.{ RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotProcessor } -import encry.view.history.History +import encry.storage.VersionalStorage.{StorageKey, StorageValue} +import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} +import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} +import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState -import encry.view.state.avlTree.{ Node, NodeSerilalizer } +import encry.view.state.avlTree.{Node, NodeSerilalizer} import encry.view.wallet.EncryWallet import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo._ @@ -34,11 +30,9 @@ import supertagged.TaggedType import scala.util.Try -class SnapshotProcessorActor( +class SnapshotProcessor( settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolder: ActorRef, - nodeViewSynchronizer: ActorRef + nodeViewHolder: ActorRef ) extends Actor with StrictLogging { @@ -46,33 +40,17 @@ class SnapshotProcessorActor( //todo 1. Add connection agreement (case while peer reconnects with other handler.ref) - var snapshotProcessor: SnapshotProcessor = - SnapshotProcessor.initialize( + var snapshotHolder: SnapshotHolder = + SnapshotHolder.initialize( settings, if (settings.snapshotSettings.enableFastSynchronization) settings.storage.state else settings.storage.snapshotHolder ) var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) var requestsProcessor: RequestsPerPeriodProcessor = RequestsPerPeriodProcessor.empty(settings) + var historyReader: HistoryReader = HistoryReader.empty - override def preStart(): Unit = - if (settings.constants.SnapshotCreationHeight <= settings.constants.MaxRollbackDepth || - (!settings.snapshotSettings.enableFastSynchronization && !settings.snapshotSettings.enableSnapshotCreation)) { - logger.info(s"Stop self(~_~)SnapshotHolder(~_~)") - context.stop(self) - } else { - context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) - logger.info(s"SnapshotHolder started.") - networkController ! RegisterMessagesHandler( - Seq( - RequestManifestMessage.NetworkMessageTypeID -> "RequestManifest", - ResponseManifestMessage.NetworkMessageTypeID -> "ResponseManifestMessage", - RequestChunkMessage.NetworkMessageTypeID -> "RequestChunkMessage", - ResponseChunkMessage.NetworkMessageTypeID -> "ResponseChunkMessage" - ), - self - ) - } + context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) override def receive: Receive = awaitingHistory @@ -227,7 +205,7 @@ class SnapshotProcessorActor( if (isValidManifest && canBeProcessed) { (for { controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotProcessor.initializeApplicableChunksCache( + processor <- snapshotHolder.initializeApplicableChunksCache( history, snapshotDownloadController.requiredManifestHeight ) @@ -238,7 +216,7 @@ class SnapshotProcessorActor( logger.debug(s"Request manifest message successfully processed.") responseManifestTimeout.foreach(_.cancel()) snapshotDownloadController = controller - snapshotProcessor = processor + snapshotHolder = processor self ! RequestNextChunks logger.debug("Manifest processed successfully.") context.become(fastSyncMod(history, none)) @@ -262,9 +240,9 @@ class SnapshotProcessorActor( def workMod(history: History): Receive = { case TreeChunks(chunks, id) => - val manifestIds: Seq[Array[Byte]] = snapshotProcessor.potentialManifestsIds + val manifestIds: Seq[Array[Byte]] = snapshotHolder.potentialManifestsIds if (!manifestIds.exists(_.sameElements(id))) { - snapshotProcessor.createNewSnapshot(ManifestId @@ id, manifestIds, chunks) + snapshotHolder.createNewSnapshot(ManifestId @@ id, manifestIds, chunks) } else logger.info(s"Doesn't need to create snapshot") case SemanticallySuccessfulModifier(block: Block) if history.isFullChainSynced => @@ -272,10 +250,10 @@ class SnapshotProcessorActor( val condition: Int = (block.header.height - settings.constants.MaxRollbackDepth) % settings.constants.SnapshotCreationHeight logger.info(s"condition = $condition") - if (condition == 0) snapshotProcessor.processNewBlock(block, history) match { + if (condition == 0) snapshotHolder.processNewBlock(block, history) match { case Left(_) => case Right(newProcessor) => - snapshotProcessor = newProcessor + snapshotHolder = newProcessor requestsProcessor = RequestsPerPeriodProcessor.empty(settings) nodeViewHolder ! RemoveRedundantManifestIds } @@ -283,8 +261,8 @@ class SnapshotProcessorActor( case DataFromPeer(message, remote) => message match { case RequestManifestMessage(requiredManifestId) - if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => - snapshotProcessor.actualManifest.foreach { m => + if requestsProcessor.canBeProcessed(snapshotHolder, requiredManifestId) => + snapshotHolder.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } @@ -294,7 +272,7 @@ class SnapshotProcessorActor( //if requestsProcessor.canProcessRequest(remote) => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") - val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) + val chunkFromDB: Option[SnapshotChunkMessage] = snapshotHolder.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) @@ -318,14 +296,14 @@ class SnapshotProcessorActor( def restartFastSync(history: History): Unit = { logger.info(s"Restart fast sync!") snapshotDownloadController = snapshotDownloadController.reInitFastSync - snapshotProcessor = snapshotProcessor.reInitStorage + snapshotHolder = snapshotHolder.reInitStorage } def timer: Option[Cancellable] = context.system.scheduler.scheduleOnce(settings.snapshotSettings.responseTimeout)(self ! CheckDelivery).some } -object SnapshotProcessorActor { +object SnapshotProcessor { case object RemoveRedundantManifestIds @@ -388,11 +366,9 @@ object SnapshotProcessorActor { ) } - def props(settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolderRef: ActorRef, - nodeViewSynchronizer: ActorRef): Props = Props( - new SnapshotProcessorActor(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) - ) + def props(settings: EncryAppSettings, nodeViewHolderRef: ActorRef): Props = + Props( + new SnapshotProcessor(settings, nodeViewHolderRef) + ) } diff --git a/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala b/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala index 2753064729..fd7d628477 100644 --- a/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala +++ b/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala @@ -24,7 +24,7 @@ object FastSyncExceptions { sealed trait SnapshotDownloadControllerException extends FastSyncException final case class InvalidManifestBytes(error: String) extends SnapshotDownloadControllerException - final case class ApplicableChunkIsAbsent(error: String, processor: SnapshotProcessor) extends FastSyncException + final case class ApplicableChunkIsAbsent(error: String, processor: SnapshotHolder) extends FastSyncException final case class BestHeaderAtHeightIsAbsent(error: String) extends FastSyncException final case class InitializeHeightAndRootKeysException(error: String) extends FastSyncException final case class ChunksIdsToDownloadException(error: String) extends FastSyncException diff --git a/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala b/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala index df3d6f393c..0ddb3dc64c 100644 --- a/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala @@ -7,7 +7,7 @@ import org.encryfoundation.common.utils.Algos final case class RequestsPerPeriodProcessor(handledRequests: Int, settings: EncryAppSettings) extends StrictLogging { - def canBeProcessed(processor: SnapshotProcessor, manifestId: Array[Byte]): Boolean = { + def canBeProcessed(processor: SnapshotHolder, manifestId: Array[Byte]): Boolean = { val actualManifestID: Option[Array[Byte]] = processor.actualManifestId logger.info(s"Requested id ${Algos.encode(manifestId)}, current manifest id ${actualManifestID.map(Algos.encode)}.") actualManifestID.exists(_.sameElements(manifestId)) diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index 588752bda5..d2635d4f21 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -8,8 +8,8 @@ import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId -import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index b0ec686518..79d3b4d1df 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,7 +1,7 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala similarity index 91% rename from src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala rename to src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index 07d25cc879..ef047908a7 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -7,13 +7,13 @@ import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.{ +import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer } -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.storage.{ RootNodesStorage, VersionalStorage } import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } @@ -38,7 +38,7 @@ import scala.collection.immutable.{ HashMap, HashSet } import scala.language.postfixOps import scala.util.{ Failure, Success } -final case class SnapshotProcessor( +final case class SnapshotHolder( settings: EncryAppSettings, storage: VersionalStorage, applicableChunks: HashSet[ByteArrayWrapper], @@ -48,24 +48,24 @@ final case class SnapshotProcessor( with SnapshotProcessorStorageAPI with AutoCloseable { - def updateCache(chunk: SnapshotChunk): SnapshotProcessor = + def updateCache(chunk: SnapshotChunk): SnapshotHolder = this.copy(chunksCache = chunksCache.updated(ByteArrayWrapper(chunk.id), chunk)) - def initializeApplicableChunksCache(history: History, height: Int): Either[FastSyncException, SnapshotProcessor] = + def initializeApplicableChunksCache(history: History, height: Int): Either[FastSyncException, SnapshotHolder] = for { stateRoot <- Either.fromOption( history.getBestHeaderAtHeight(height).map(_.stateRoot), BestHeaderAtHeightIsAbsent(s"There is no best header at required height $height") ) - processor: SnapshotProcessor = this.copy(applicableChunks = HashSet(ByteArrayWrapper(stateRoot))) + processor: SnapshotHolder = this.copy(applicableChunks = HashSet(ByteArrayWrapper(stateRoot))) resultedProcessor <- processor.initializeHeightAndRootKeys(stateRoot, height) match { case Left(error) => - InitializeHeightAndRootKeysException(error.getMessage).asLeft[SnapshotProcessor] + InitializeHeightAndRootKeysException(error.getMessage).asLeft[SnapshotHolder] case Right(newProcessor) => newProcessor.asRight[FastSyncException] } } yield resultedProcessor - private def initializeHeightAndRootKeys(rootNodeId: Array[Byte], height: Int): Either[Throwable, SnapshotProcessor] = + private def initializeHeightAndRootKeys(rootNodeId: Array[Byte], height: Int): Either[Throwable, SnapshotHolder] = Either.catchNonFatal { storage.insert( StorageVersion @@ Random.randomBytes(), @@ -75,7 +75,7 @@ final case class SnapshotProcessor( this } - def reInitStorage: SnapshotProcessor = + def reInitStorage: SnapshotHolder = try { storage.close() wallet.foreach(_.close()) @@ -84,7 +84,7 @@ final case class SnapshotProcessor( import org.apache.commons.io.FileUtils FileUtils.deleteDirectory(stateDir) FileUtils.deleteDirectory(walletDir) - SnapshotProcessor.initialize(settings, settings.storage.state) + SnapshotHolder.initialize(settings, settings.storage.state) } catch { case err: Throwable => throw new Exception(s"Exception ${err.getMessage} has occurred while restarting fast sync process") @@ -98,7 +98,7 @@ final case class SnapshotProcessor( case emptyNode: EmptyNode[StorageKey, StorageValue] => List.empty[Node[StorageKey, StorageValue]] } - def applyChunk(chunk: SnapshotChunk): Either[ChunkApplyError, SnapshotProcessor] = { + def applyChunk(chunk: SnapshotChunk): Either[ChunkApplyError, SnapshotHolder] = { val kSerializer: Serializer[StorageKey] = implicitly[Serializer[StorageKey]] val vSerializer: Serializer[StorageValue] = implicitly[Serializer[StorageValue]] val nodes: List[Node[StorageKey, StorageValue]] = flatten(chunk.node) @@ -154,7 +154,7 @@ final case class SnapshotProcessor( val newApplicableChunk = (applicableChunks -- toStorage.map(node => ByteArrayWrapper(node.hash))) ++ toApplicable.map(node => ByteArrayWrapper(node.hash)) this.copy(applicableChunks = newApplicableChunk).asRight[ChunkApplyError] - case Left(exception) => ChunkApplyError(exception.getMessage).asLeft[SnapshotProcessor] + case Left(exception) => ChunkApplyError(exception.getMessage).asLeft[SnapshotHolder] } } @@ -165,7 +165,7 @@ final case class SnapshotProcessor( s"Node hash:(${Algos.encode(chunk.node.hash)}) doesn't equal to chunk id:(${Algos.encode(chunk.id)})" ).asLeft[SnapshotChunk] - private def getNextApplicableChunk: Either[FastSyncException, (SnapshotChunk, SnapshotProcessor)] = + private def getNextApplicableChunk: Either[FastSyncException, (SnapshotChunk, SnapshotHolder)] = for { idAndChunk <- Either.fromOption(chunksCache.find { case (id, _) => applicableChunks.contains(id) }, ApplicableChunkIsAbsent("There are no applicable chunks in cache", this)) @@ -176,7 +176,7 @@ final case class SnapshotProcessor( (chunk, this.copy(chunksCache = newChunksCache)) } - def processNextApplicableChunk(snapshotProcessor: SnapshotProcessor): Either[FastSyncException, SnapshotProcessor] = + def processNextApplicableChunk(snapshotProcessor: SnapshotHolder): Either[FastSyncException, SnapshotHolder] = for { chunkAndProcessor <- snapshotProcessor.getNextApplicableChunk (chunk, processor) = chunkAndProcessor @@ -214,7 +214,7 @@ final case class SnapshotProcessor( node <- getNode(rootNodeId) } yield node - def processNewBlock(block: Block, history: History): Either[ProcessNewBlockError, SnapshotProcessor] = { + def processNewBlock(block: Block, history: History): Either[ProcessNewBlockError, SnapshotHolder] = { logger.info( s"Start updating actual manifest to new one at height " + s"${block.header.height} with block id ${block.encodedId}." @@ -226,7 +226,7 @@ final case class SnapshotProcessor( id: ManifestId, manifestIds: Seq[Array[Byte]], newChunks: List[SnapshotChunk] - ): Either[ProcessNewSnapshotError, SnapshotProcessor] = { + ): Either[ProcessNewSnapshotError, SnapshotHolder] = { //todo add only exists chunks val manifest: SnapshotManifest = SnapshotManifest(id, newChunks.map(_.id)) val snapshotToDB: List[(StorageKey, StorageValue)] = newChunks.map { elem => @@ -242,12 +242,12 @@ final case class SnapshotProcessor( val toApply: List[(StorageKey, StorageValue)] = manifestToDB :: updateList :: snapshotToDB logger.info(s"A new snapshot created successfully. Insertion started.") Either.catchNonFatal(storage.insert(StorageVersion @@ Random.randomBytes(), toApply, List.empty)) match { - case Left(value) => ProcessNewSnapshotError(value.getMessage).asLeft[SnapshotProcessor] + case Left(value) => ProcessNewSnapshotError(value.getMessage).asLeft[SnapshotHolder] case Right(_) => this.asRight[ProcessNewSnapshotError] } } - private def updateActualSnapshot(history: History, height: Int): Either[ProcessNewBlockError, SnapshotProcessor] = + private def updateActualSnapshot(history: History, height: Int): Either[ProcessNewBlockError, SnapshotHolder] = for { bestManifestId <- Either.fromOption( history.getBestHeaderAtHeight(height).map(header => Algos.hash(header.stateRoot ++ header.id)), @@ -269,7 +269,7 @@ final case class SnapshotProcessor( Either.catchNonFatal( storage.insert(StorageVersion @@ Random.randomBytes(), List(toApply), toDelete.map(StorageKey @@ _)) ) match { - case Left(error) => ProcessNewBlockError(error.getMessage).asLeft[SnapshotProcessor] + case Left(error) => ProcessNewBlockError(error.getMessage).asLeft[SnapshotHolder] case Right(_) => this.asRight[ProcessNewBlockError] } } @@ -278,15 +278,15 @@ final case class SnapshotProcessor( override def close(): Unit = storage.close() } -object SnapshotProcessor extends StrictLogging { +object SnapshotHolder extends StrictLogging { - def initialize(settings: EncryAppSettings, storageType: StorageType): SnapshotProcessor = + def initialize(settings: EncryAppSettings, storageType: StorageType): SnapshotHolder = if (settings.snapshotSettings.enableFastSynchronization) create(settings, new File(s"${settings.directory}/state"), storageType) else create(settings, getDirProcessSnapshots(settings), storageType) - def recreateAfterFastSyncIsDone(settings: EncryAppSettings): SnapshotProcessor = { + def recreateAfterFastSyncIsDone(settings: EncryAppSettings): SnapshotHolder = { val snapshotStorage = getDirProcessSnapshots(settings) snapshotStorage.mkdirs() val storage: VersionalStorage = @@ -299,12 +299,12 @@ object SnapshotProcessor extends StrictLogging { val levelDBInit: DB = LevelDbFactory.factory.open(snapshotStorage, new Options) VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) } - new SnapshotProcessor(settings, storage, HashSet.empty, HashMap.empty, none[EncryWallet]) + new SnapshotHolder(settings, storage, HashSet.empty, HashMap.empty, none[EncryWallet]) } def getDirProcessSnapshots(settings: EncryAppSettings): File = new File(s"${settings.directory}/snapshots") - def create(settings: EncryAppSettings, snapshotsDir: File, storageType: StorageType): SnapshotProcessor = { + def create(settings: EncryAppSettings, snapshotsDir: File, storageType: StorageType): SnapshotHolder = { snapshotsDir.mkdirs() val storage: VersionalStorage = storageType match { @@ -328,6 +328,6 @@ object SnapshotProcessor extends StrictLogging { .some else none[EncryWallet] - new SnapshotProcessor(settings, storage, HashSet.empty, HashMap.empty, wallet) + new SnapshotHolder(settings, storage, HashSet.empty, HashMap.empty, wallet) } } diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index c0f868cbb3..f52505f78c 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,7 +3,7 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.{ SnapshotManifest, SnapshotManifestSerializer } +import encry.nvg.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 983cd6317a..9b97e83d45 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,8 +4,8 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotChunk -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.SnapshotChunk +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} import encry.view.state.UtxoState From abb6008c2fff3c84d9c429b48cf6e220c35619db Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 11:03:51 +0300 Subject: [PATCH 049/119] add processing getPeers message on pk --- src/main/scala/encry/network/DM.scala | 6 ++- .../scala/encry/network/MessageBuilder.scala | 44 ++++++++++++------- src/main/scala/encry/network/Messages.scala | 1 + .../scala/encry/network/NetworkRouter.scala | 5 ++- src/main/scala/encry/network/PK.scala | 7 ++- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 965e62d027..2b5e86a237 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -4,7 +4,7 @@ import java.net.InetSocketAddress import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging -import encry.network.DM.{AwaitingRequest, RequestSent} +import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork @@ -47,6 +47,9 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) + case IsRequested(modId) => + logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") + sender() ! receivedModifier.contains(toKey(modId)) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) @@ -57,6 +60,5 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) case class IsRequested(modifierId: ModifierId) - def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 171ac31988..3ee27506b9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -11,16 +11,16 @@ import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent} import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos +import scala.concurrent.Await import scala.concurrent.duration._ import scala.util.Try -case class MessageBuilder(msg: MessageToNetwork, - peersKeeper: ActorRef, +case class MessageBuilder(peersKeeper: ActorRef, deliveryManager: ActorRef) extends Actor with StrictLogging { import context.dispatcher @@ -31,13 +31,21 @@ case class MessageBuilder(msg: MessageToNetwork, case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + logger.info(s"Going to req mods from ${peer.socketAddress}") modsIds.foreach { modId => - for { - isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (!isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + val res123 = (deliveryManager ? IsRequested(modId)) + //Await.result(res123, 2 minutes) + logger.info(s"res: ${res123.getClass}") + res123.mapTo[Boolean].foreach { res => + if (res) logger.info(s"Another res: ${res}") + } + (deliveryManager ? IsRequested(modId)).mapTo[Boolean].foreach { isRequested => + logger.info(s"isRequested: ${isRequested}") + if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } } } } @@ -48,7 +56,7 @@ case class MessageBuilder(msg: MessageToNetwork, modsIds.foreach { modId => for { isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (!isRequested) { + } yield if (isRequested) { peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") @@ -58,7 +66,7 @@ case class MessageBuilder(msg: MessageToNetwork, context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => - peers.foreach(_.handlerRef ! syncInfo) + peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) } context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => @@ -73,6 +81,13 @@ case class MessageBuilder(msg: MessageToNetwork, peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) } context.stop(self) + case SendPeers(peers, remote) => + Try { + (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! PeersNetworkMessage(peers) + } + } + } } @@ -102,7 +117,6 @@ object MessageBuilder { override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger } - def props(msg: MessageToNetwork, - peersKeeper: ActorRef, - deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) + def props(peersKeeper: ActorRef, + deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 10f881290e..a66b748415 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -17,5 +17,6 @@ object Messages { modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork final case class BroadcastModifier(modifierTypeId: ModifierTypeId, modifierId: ModifierId) extends MessageToNetwork + final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 8d95e93c67..6d2b77b758 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -67,7 +67,10 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => - context.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") + context.actorOf( + MessageBuilder.props(peersKeeper, deliveryManager), + s"messageBuilder${Random.nextInt()}" + ) ! msg } def peersLogic: Receive = { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 6f0067b68e..8a87c7cb19 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -8,6 +8,7 @@ import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} +import encry.network.Messages.MessageToNetwork.SendPeers import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection @@ -157,12 +158,10 @@ class PK(networkSettings: NetworkSettings, else add.getAddress.isSiteLocalAddress && add != remote }.getOrElse(false) - val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) + val peers: List[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote).toList logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") logger.info(s"Remote is side local: ${remote} : ${Try(remote.getAddress.isSiteLocalAddress)}") - connectedPeers.getAll.find(_._1 == remote).foreach { - case (_, info) => info.connectedPeer.handlerRef ! PeersNetworkMessage(peers) - } + context.parent ! SendPeers(peers, remote) } } From 9ec8d6e59d20c1f7e120fe9db121d73faf292e77 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:35:14 +0300 Subject: [PATCH 050/119] simple pipeline of mods downloading --- src/main/scala/encry/network/DM.scala | 26 ++++++++++--- .../scala/encry/network/MessageBuilder.scala | 39 ++++++------------- .../scala/encry/network/NetworkRouter.scala | 2 + 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2b5e86a237..3a938ce0bc 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -3,10 +3,11 @@ package encry.network import java.net.InetSocketAddress import akka.actor.{Actor, Props} +import akka.pattern._ import com.typesafe.scalalogging.StrictLogging -import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent} +import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent, RequestStatus} import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.ModifierFromNetwork import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage @@ -16,6 +17,7 @@ import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable +import scala.concurrent.Future case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { @@ -30,6 +32,14 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) override def receive: Receive = { + case DataFromPeer(ModifiersNetworkMessage(data), source) => + data._2.foreach { case (id, bytes) => + if (expectedModifiers.contains(toKey(id))) { + context.parent ! ModifierFromNetwork(source, data._1, id, bytes) + expectedModifiers -= toKey(id) + receivedModifier += toKey(id) + } else logger.info("Receive spam!") + } case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! @@ -47,9 +57,12 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) - case IsRequested(modId) => - logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") - sender() ! receivedModifier.contains(toKey(modId)) + case IsRequested(modIds) => + //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") + sender ! RequestStatus( + modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + ) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) @@ -59,6 +72,7 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) - case class IsRequested(modifierId: ModifierId) + case class IsRequested(modifiersId: List[ModifierId]) + case class RequestStatus(requested: List[ModifierId], notRequested: List[ModifierId]) def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 3ee27506b9..7740ff4b92 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -8,15 +8,13 @@ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo -import encry.network.DM.{IsRequested, RequestSent} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} -import encry.network.Messages.MessageToNetwork +import encry.network.DM.{IsRequested, RequestSent, RequestStatus} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos -import scala.concurrent.Await import scala.concurrent.duration._ import scala.util.Try @@ -30,36 +28,22 @@ case class MessageBuilder(peersKeeper: ActorRef, override def receive: Receive = { case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { - (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => logger.info(s"Going to req mods from ${peer.socketAddress}") - modsIds.foreach { modId => - val res123 = (deliveryManager ? IsRequested(modId)) - //Await.result(res123, 2 minutes) - logger.info(s"res: ${res123.getClass}") - res123.mapTo[Boolean].foreach { res => - if (res) logger.info(s"Another res: ${res}") - } - (deliveryManager ? IsRequested(modId)).mapTo[Boolean].foreach { isRequested => - logger.info(s"isRequested: ${isRequested}") - if (!isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") - } + (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) + modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } - context.stop(self) case RequestFromLocal(None, modTypeId, modsIds) => Try { (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => - modsIds.foreach { modId => - for { - isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) + modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } @@ -94,6 +78,7 @@ case class MessageBuilder(peersKeeper: ActorRef, object MessageBuilder { case object GetPeers + case class MsgSent(msgType: Byte, receiver: InetSocketAddress) case class GetPeerInfo(peerIp: InetSocketAddress) trait GetPeerByPredicate { diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 6d2b77b758..a379da3e75 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -9,6 +9,7 @@ import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -43,6 +44,7 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) messagesHandlers += (ids -> handler) + case msg: MsgSent => context.stop(sender()) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") case RegisterForModsHandling => handlerForMods = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") From 72220c3fb74992b84a1f717a1fc2d67cfd5618c0 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 13:39:44 +0300 Subject: [PATCH 051/119] snapshot processor frame --- src/main/scala/encry/network/Messages.scala | 1 + src/main/scala/encry/nvg/NodeViewHolder.scala | 15 - .../scala/encry/nvg/SnapshotDownloader.scala | 285 +++++++++------ .../scala/encry/nvg/SnapshotProcessor.scala | 336 +++++++++--------- .../encry/view/history/HistoryReader.scala | 13 + 5 files changed, 357 insertions(+), 293 deletions(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index a66b748415..a9303937eb 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -18,5 +18,6 @@ object Messages { final case class BroadcastModifier(modifierTypeId: ModifierTypeId, modifierId: ModifierId) extends MessageToNetwork final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork + final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f62007ec6e..a7c6fd5a01 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -339,21 +339,6 @@ class NodeViewHolder( case _: Header => true case _: Payload => false }, success = true) - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && - modifier.modifierTypeId == Payload.modifierTypeId && - historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info( - s"getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}. " + - s"heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" - ) - historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { - h: Header => - context.parent ! RequiredManifestHeightAndId( - historyBeforeStUpdate.lastAvailableManifestHeight, - Algos.hash(h.stateRoot ++ h.id) - ) - } - } logger.info( s"Going to apply modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the state. " + s"Progress info is: $progressInfo." diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/SnapshotDownloader.scala index 408d1473b9..902bba67c3 100644 --- a/src/main/scala/encry/nvg/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/SnapshotDownloader.scala @@ -1,23 +1,32 @@ package encry.nvg -import akka.actor.{Actor, Cancellable} -import cats.syntax.option._ +import akka.actor.{ Actor, Cancellable } import cats.syntax.either._ +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidStateAfterFastSync} +import encry.network.BlackList.BanReason.{ InvalidChunkMessage, InvalidStateAfterFastSync } +import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer -import encry.nvg.SnapshotProcessor.{CheckDelivery, FastSyncDone, FastSyncFinished, RequestNextChunks, RequiredManifestHeightAndId} +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} -import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} -import encry.view.history.{History, HistoryReader} +import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } +import encry.view.fast.sync.{ SnapshotDownloadController, SnapshotHolder } +import encry.view.history.{ History, HistoryReader } import encry.view.wallet.EncryWallet -import org.encryfoundation.common.network.BasicMessagesRepo.{RequestChunkMessage, ResponseChunkMessage, ResponseManifestMessage} +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + RequestChunkMessage, + ResponseChunkMessage, + ResponseManifestMessage +} import org.encryfoundation.common.utils.Algos class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { + import context.dispatcher + var snapshotHolder: SnapshotHolder = SnapshotHolder.initialize( settings, @@ -25,116 +34,172 @@ class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLo else settings.storage.snapshotHolder ) var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) - var historyReader: HistoryReader = HistoryReader.empty + var historyReader: HistoryReader = HistoryReader.empty + var requiredManifestHeight: Int = 0 override def receive: Receive = ??? - - def fastSyncMod( - history: History, - responseTimeout: Option[Cancellable] - ): Receive = { - case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") - message match { - case ResponseManifestMessage(manifest) => - logger.info( - s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." - ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotHolder.validateChunkId(chunk) - processor = snapshotHolder.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotHolder] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotHolder = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + def receive1: Receive = { + case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Payload.modifierTypeId => + val bestBlockHeight: Int = historyReader.getBestBlockHeight + if (historyReader.isFastSyncInProcess && bestBlockHeight >= requiredManifestHeight) { + logger.info( + s"Snapshot downloader got new block. Current best block height is: $bestBlockHeight. " + + s"Height of last available payload for request is: $requiredManifestHeight." + ) + historyReader + .getBestHeaderAtHeight(requiredManifestHeight) + .map { h: Header => + RequiredManifestHeightAndId( + requiredManifestHeight, + Algos.hash(h.stateRoot ++ h.id) + ) } - - case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote}") - - case _ => - } - - case RequestNextChunks => - responseTimeout.foreach(_.cancel()) - (for { - controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController - _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") - } yield controllerAndIds) match { - case Left(err) => - logger.info(s"Error has occurred: ${err.error}") - throw new Exception(s"Error has occurred: ${err.error}") - case Right(controllerAndIds) => - snapshotDownloadController = controllerAndIds._1 - controllerAndIds._2.foreach { msg => - // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => - // peer.handlerRef ! msg - // } + .foreach { manifestToId: RequiredManifestHeightAndId => + logger.info( + s"Manifest height is: ${manifestToId.height}. " + + s"Manifest id is: ${Algos.encode(manifestToId.manifestId)}" + ) + snapshotDownloadController = snapshotDownloadController.copy( + requiredManifestHeight = manifestToId.height, + requiredManifestId = manifestToId.manifestId + ) } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) + restartFastSync() + self ! BroadcastManifestRequestMessage } - - case RequiredManifestHeightAndId(height, manifestId) => + case SemanticallySuccessfulModifier(_) => + case BroadcastManifestRequestMessage => logger.info( - s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + - s"New required manifest id is ${Algos.encode(manifestId)}." - ) - snapshotDownloadController = snapshotDownloadController.copy( - requiredManifestHeight = height, - requiredManifestId = manifestId + s"Snapshot downloader got BroadcastManifestRequestMessage message. " + + s"Required manifest id is: ${Algos.encode(snapshotDownloadController.requiredManifestId)}." ) - restartFastSync(history) - self ! BroadcastManifestRequestMessage - context.become(awaitManifestMod(none, history).orElse(commonMessages)) + context.parent ! BroadcastManifestRequest(snapshotDownloadController.requiredManifestId) - case CheckDelivery => - snapshotDownloadController.awaitedChunks.map { id => - RequestChunkMessage(id.data) - }.foreach { msg => - //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) + val newScheduler: Cancellable = + context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { + logger.info(s"Trigger scheduler for re-request manifest") + self ! BroadcastManifestRequestMessage + } + logger.info(s"Start awaiting manifest network message.") +// context.become(awaitingManifestFromNetwork(newScheduler).orElse(commonMessages)) + } - case FastSyncDone => - if (settings.snapshotSettings.enableSnapshotCreation) { - logger.info(s"Snapshot holder context.become to snapshot processing") - snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) - snapshotDownloadController.storage.close() - context.system.scheduler - .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) - context.become(workMod(history).orElse(commonMessages)) - } else { - logger.info(s"Stop processing snapshots") - context.stop(self) - } + def awaitingManifestFromNetwork(scheduler: Cancellable): Receive = { + case _ => + } + +// def fastSyncMod( +// history: History, +// responseTimeout: Option[Cancellable] +// ): Receive = { +// case DataFromPeer(message, remote) => +// logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") +// message match { +// case ResponseManifestMessage(manifest) => +// logger.info( +// s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." +// ) +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotHolder.validateChunkId(chunk) +// processor = snapshotHolder.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotHolder] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotHolder = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } +// +// case ResponseChunkMessage(_) => +// logger.info(s"Received chunk from unexpected peer ${remote}") +// +// case _ => +// } +// +// case RequestNextChunks => +// responseTimeout.foreach(_.cancel()) +// (for { +// controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController +// _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") +// } yield controllerAndIds) match { +// case Left(err) => +// logger.info(s"Error has occurred: ${err.error}") +// throw new Exception(s"Error has occurred: ${err.error}") +// case Right(controllerAndIds) => +// snapshotDownloadController = controllerAndIds._1 +// controllerAndIds._2.foreach { msg => +// // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +// // peer.handlerRef ! msg +// // } +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// } +// +// case RequiredManifestHeightAndId(height, manifestId) => +// logger.info( +// s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + +// s"New required manifest id is ${Algos.encode(manifestId)}." +// ) +// snapshotDownloadController = snapshotDownloadController.copy( +// requiredManifestHeight = height, +// requiredManifestId = manifestId +// ) +// restartFastSync(history) +// self ! BroadcastManifestRequestMessage +// context.become(awaitManifestMod(none, history).orElse(commonMessages)) +// +// case CheckDelivery => +// snapshotDownloadController.awaitedChunks.map { id => +// RequestChunkMessage(id.data) +// }.foreach { msg => +// //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// +// case FastSyncDone => +// if (settings.snapshotSettings.enableSnapshotCreation) { +// logger.info(s"Snapshot holder context.become to snapshot processing") +// snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) +// snapshotDownloadController.storage.close() +// context.system.scheduler +// .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) +// context.become(workMod(history).orElse(commonMessages)) +// } else { +// logger.info(s"Stop processing snapshots") +// context.stop(self) +// } +// } + + def restartFastSync(): Unit = { + logger.info(s"Restart fast sync!") + snapshotDownloadController = snapshotDownloadController.reInitFastSync + snapshotHolder = snapshotHolder.reInitStorage } } diff --git a/src/main/scala/encry/nvg/SnapshotProcessor.scala b/src/main/scala/encry/nvg/SnapshotProcessor.scala index bc52858361..ed002d7f03 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessor.scala @@ -59,7 +59,7 @@ class SnapshotProcessor( if (settings.snapshotSettings.enableFastSynchronization && !history.isBestBlockDefined && !settings.node.offlineGeneration) { logger.info(s"Start in fast sync regime") - context.become(fastSyncMod(history, none).orElse(commonMessages)) +// context.become(fastSyncMod(history, none).orElse(commonMessages)) } else { logger.info(s"Start in snapshot processing regime") context.system.scheduler @@ -69,174 +69,174 @@ class SnapshotProcessor( case nonsense => logger.info(s"Snapshot holder got $nonsense while history awaiting") } - def fastSyncMod( - history: History, - responseTimeout: Option[Cancellable] - ): Receive = { - case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") - message match { - case ResponseManifestMessage(manifest) => - logger.info( - s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." - ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotProcessor.validateChunkId(chunk) - processor = snapshotProcessor.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotProcessor] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotProcessor = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks - } - - case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote}") - - case _ => - } - - case RequestNextChunks => - responseTimeout.foreach(_.cancel()) - (for { - controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController - _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") - } yield controllerAndIds) match { - case Left(err) => - logger.info(s"Error has occurred: ${err.error}") - throw new Exception(s"Error has occurred: ${err.error}") - case Right(controllerAndIds) => - snapshotDownloadController = controllerAndIds._1 - controllerAndIds._2.foreach { msg => -// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => -// peer.handlerRef ! msg +// def fastSyncMod( +// history: History, +// responseTimeout: Option[Cancellable] +// ): Receive = { +// case DataFromPeer(message, remote) => +// logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") +// message match { +// case ResponseManifestMessage(manifest) => +// logger.info( +// s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." +// ) +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotProcessor.validateChunkId(chunk) +// processor = snapshotProcessor.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotProcessor] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotProcessor = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } +// +// case ResponseChunkMessage(_) => +// logger.info(s"Received chunk from unexpected peer ${remote}") +// +// case _ => +// } +// +// case RequestNextChunks => +// responseTimeout.foreach(_.cancel()) +// (for { +// controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController +// _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") +// } yield controllerAndIds) match { +// case Left(err) => +// logger.info(s"Error has occurred: ${err.error}") +// throw new Exception(s"Error has occurred: ${err.error}") +// case Right(controllerAndIds) => +// snapshotDownloadController = controllerAndIds._1 +// controllerAndIds._2.foreach { msg => +//// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +//// peer.handlerRef ! msg +//// } +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// } +// +// case RequiredManifestHeightAndId(height, manifestId) => +//// logger.info( +//// s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + +//// s"New required manifest id is ${Algos.encode(manifestId)}." +//// ) +//// snapshotDownloadController = snapshotDownloadController.copy( +//// requiredManifestHeight = height, +//// requiredManifestId = manifestId +//// ) +// restartFastSync(history) +// self ! BroadcastManifestRequestMessage +//// context.become(awaitManifestMod(none, history).orElse(commonMessages)) +// +// case CheckDelivery => +// snapshotDownloadController.awaitedChunks.map { id => +// RequestChunkMessage(id.data) +// }.foreach { msg => +// //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// +// case FastSyncDone => +// if (settings.snapshotSettings.enableSnapshotCreation) { +// logger.info(s"Snapshot holder context.become to snapshot processing") +// snapshotProcessor = SnapshotProcessor.recreateAfterFastSyncIsDone(settings) +// snapshotDownloadController.storage.close() +// context.system.scheduler +// .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) +// context.become(workMod(history).orElse(commonMessages)) +// } else { +// logger.info(s"Stop processing snapshots") +// context.stop(self) +// } +// } +// +// def awaitManifestMod( +// responseManifestTimeout: Option[Cancellable], +// history: History +// ): Receive = { +// case BroadcastManifestRequestMessage => +//// logger.info( +//// s"Snapshot holder got HeaderChainIsSynced. Broadcasts request for new manifest with id " + +//// s"${Algos.encode(snapshotDownloadController.requiredManifestId)}" +//// ) +//// nodeViewSynchronizer ! SendToNetwork(RequestManifestMessage(snapshotDownloadController.requiredManifestId), +//// Broadcast) +//// val newScheduler = context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { +//// logger.info(s"Trigger scheduler for re-request manifest") +//// self ! BroadcastManifestRequestMessage +//// } +//// logger.info(s"Start awaiting manifest network message.") +//// context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) +// +// case DataFromPeer(message, remote) => +// message match { +// case ResponseManifestMessage(manifest) => +// val isValidManifest: Boolean = +// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) +// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed +// if (isValidManifest && canBeProcessed) { +// (for { +// controller <- snapshotDownloadController.processManifest(manifest, remote, history) +// processor <- snapshotHolder.initializeApplicableChunksCache( +// history, +// snapshotDownloadController.requiredManifestHeight +// ) +// } yield (controller, processor)) match { +// case Left(error) => +// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) +// case Right((controller, processor)) => +// logger.debug(s"Request manifest message successfully processed.") +// responseManifestTimeout.foreach(_.cancel()) +// snapshotDownloadController = controller +// snapshotHolder = processor +// self ! RequestNextChunks +// logger.debug("Manifest processed successfully.") +// context.become(fastSyncMod(history, none)) // } - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) - } - - case RequiredManifestHeightAndId(height, manifestId) => - logger.info( - s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + - s"New required manifest id is ${Algos.encode(manifestId)}." - ) - snapshotDownloadController = snapshotDownloadController.copy( - requiredManifestHeight = height, - requiredManifestId = manifestId - ) - restartFastSync(history) - self ! BroadcastManifestRequestMessage - context.become(awaitManifestMod(none, history).orElse(commonMessages)) - - case CheckDelivery => - snapshotDownloadController.awaitedChunks.map { id => - RequestChunkMessage(id.data) - }.foreach { msg => - //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) - - case FastSyncDone => - if (settings.snapshotSettings.enableSnapshotCreation) { - logger.info(s"Snapshot holder context.become to snapshot processing") - snapshotProcessor = SnapshotProcessor.recreateAfterFastSyncIsDone(settings) - snapshotDownloadController.storage.close() - context.system.scheduler - .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) - context.become(workMod(history).orElse(commonMessages)) - } else { - logger.info(s"Stop processing snapshots") - context.stop(self) - } - } - - def awaitManifestMod( - responseManifestTimeout: Option[Cancellable], - history: History - ): Receive = { - case BroadcastManifestRequestMessage => - logger.info( - s"Snapshot holder got HeaderChainIsSynced. Broadcasts request for new manifest with id " + - s"${Algos.encode(snapshotDownloadController.requiredManifestId)}" - ) - nodeViewSynchronizer ! SendToNetwork(RequestManifestMessage(snapshotDownloadController.requiredManifestId), - Broadcast) - val newScheduler = context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { - logger.info(s"Trigger scheduler for re-request manifest") - self ! BroadcastManifestRequestMessage - } - logger.info(s"Start awaiting manifest network message.") - context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) - - case DataFromPeer(message, remote) => - message match { - case ResponseManifestMessage(manifest) => - val isValidManifest: Boolean = - snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) - val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed - if (isValidManifest && canBeProcessed) { - (for { - controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotHolder.initializeApplicableChunksCache( - history, - snapshotDownloadController.requiredManifestHeight - ) - } yield (controller, processor)) match { - case Left(error) => - nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) - case Right((controller, processor)) => - logger.debug(s"Request manifest message successfully processed.") - responseManifestTimeout.foreach(_.cancel()) - snapshotDownloadController = controller - snapshotHolder = processor - self ! RequestNextChunks - logger.debug("Manifest processed successfully.") - context.become(fastSyncMod(history, none)) - } - } else if (!isValidManifest) { - logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") - nodeViewSynchronizer ! BanPeer( - remote, - InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") - ) - } else logger.info(s"Doesn't need to process new manifest.") - case _ => - } - - case msg @ RequiredManifestHeightAndId(_, _) => - self ! msg - responseManifestTimeout.foreach(_.cancel()) - logger.info(s"Got RequiredManifestHeightAndId while awaitManifestMod") - context.become(fastSyncMod(history, none)) - } +// } else if (!isValidManifest) { +// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") +// nodeViewSynchronizer ! BanPeer( +// remote, +// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") +// ) +// } else logger.info(s"Doesn't need to process new manifest.") +// case _ => +// } +// +// case msg @ RequiredManifestHeightAndId(_, _) => +// self ! msg +// responseManifestTimeout.foreach(_.cancel()) +// logger.info(s"Got RequiredManifestHeightAndId while awaitManifestMod") +// context.become(fastSyncMod(history, none)) +// } def workMod(history: History): Receive = { case TreeChunks(chunks, id) => diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index e4f652d869..8e3deb193e 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,6 +1,7 @@ package encry.view.history import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } +import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -10,6 +11,10 @@ trait HistoryReader { def getBestHeaderHeight: Int + def getBestBlockHeight: Int + + def getBestHeaderAtHeight(h: Int): Option[Header] + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] def compare(si: SyncInfo): HistoryComparisonResult @@ -25,28 +30,36 @@ trait HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] def syncInfo: SyncInfo + + def isFastSyncInProcess: Boolean } object HistoryReader { def empty: HistoryReader = new HistoryReader { def isModifierDefined(id: ModifierId): Boolean = false def getBestHeaderHeight: Int = -1 + def getBestBlockHeight: Int = -1 + def getBestHeaderAtHeight(h: Int): Option[Header] = None def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty var isFullChainSynced: Boolean = false def compare(si: SyncInfo): HistoryComparisonResult = Older def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty def syncInfo: SyncInfo = SyncInfo(Seq.empty) + def isFastSyncInProcess: Boolean = false } def apply(history: History): HistoryReader = new HistoryReader { def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) def getBestHeaderHeight: Int = history.getBestHeaderHeight + def getBestBlockHeight: Int = history.getBestBlockHeight + def getBestHeaderAtHeight(h: Int): Option[Header] = history.getBestHeaderAtHeight(h) def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) var isFullChainSynced: Boolean = history.isFullChainSynced def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) def syncInfo: SyncInfo = history.syncInfo + def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal } } From fcb9d81d5835399f474e3abcfcca5a36de95b28c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:40:06 +0300 Subject: [PATCH 052/119] add removing app.mod from expected modifier --- src/main/scala/encry/network/NetworkRouter.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a379da3e75..d1b5968234 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -17,6 +17,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -66,6 +67,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: SemanticallySuccessfulModifier => deliveryManager ! msg case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => From 309a96b3eb0d9f89a9349dd414bf3283962f0ea2 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 13:46:35 +0300 Subject: [PATCH 053/119] fixed scala match error --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 17d755f257..8415387d83 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -62,6 +62,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St block.encodedId -> toProto(block.header), block.payload.encodedId -> toProto(block.payload) ) + case _ => } case DataFromPeer(message, remote) => message match { From e1caeca743bead87921d30c02897b8aa33098100 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:58:23 +0300 Subject: [PATCH 054/119] add logging --- src/main/scala/encry/network/DM.scala | 7 +++++-- src/main/scala/encry/network/MessageBuilder.scala | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 3a938ce0bc..889a14f7b0 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -45,8 +45,11 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! AwaitingRequest(peer, modTypeId, modId, 1) ) - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => - if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> + context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, attempts + 1) + ) case AwaitingRequest(peer, _, modId, _) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 7740ff4b92..60b5f315a9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -33,6 +33,7 @@ case class MessageBuilder(peersKeeper: ActorRef, (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } From 5b68110f0fe79a36cda8508c8c30e776895bb807 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 14:11:36 +0300 Subject: [PATCH 055/119] logging improved --- src/main/scala/encry/nvg/NodeViewHolder.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index a7c6fd5a01..bfed745d26 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -77,14 +77,16 @@ class NodeViewHolder( context.system.scheduler.schedule(1.seconds, 10.seconds) { logger.info( - s"History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + s"State height is: ${nodeView.state.height}.\n " + - s"Cache size is: ${ModifiersCache.size}.\n " + - s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." + s"Cache size is: ${ModifiersCache.size}.\n " + ) + logger.info( + s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." ) } From 956127be1a078bf194760bb17b8d3a7c78abf455 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 14:11:47 +0300 Subject: [PATCH 056/119] send info about request to delivery manager --- src/main/scala/encry/network/MessageBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 60b5f315a9..ea5439805c 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -32,7 +32,7 @@ case class MessageBuilder(peersKeeper: ActorRef, logger.info(s"Going to req mods from ${peer.socketAddress}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) - modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } From c2c7e68342a82898dae27b0ed7824ff9d4545cbd Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 14:18:27 +0300 Subject: [PATCH 057/119] provide stopping msgBuilder after msg sending --- src/main/scala/encry/network/MessageBuilder.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index ea5439805c..941cb03de1 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -52,24 +52,27 @@ case class MessageBuilder(peersKeeper: ActorRef, case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) + context.parent ! MsgSent(SyncInfoNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } - context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) + context.parent ! MsgSent(ModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } context.stop(self) case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) + context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } context.stop(self) case SendPeers(peers, remote) => Try { (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! PeersNetworkMessage(peers) + context.parent ! MsgSent(PeersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } From 5572e400f2fc4131e093e91fcd713fd869d7ba9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 14:50:31 +0300 Subject: [PATCH 058/119] code style updated --- src/main/scala/encry/nvg/ModifiersCache.scala | 114 ++++++++++-------- src/main/scala/encry/nvg/NodeViewHolder.scala | 23 +--- 2 files changed, 69 insertions(+), 68 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 97e4d5b21c..07ed709065 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -2,7 +2,7 @@ package encry.nvg import com.typesafe.scalalogging.StrictLogging import encry.view.history.History -import encry.view.history.ValidationError.{FatalValidationError, NonFatalValidationError} +import encry.view.history.ValidationError.{ FatalValidationError, NonFatalValidationError } import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos @@ -17,7 +17,7 @@ object ModifiersCache extends StrictLogging { private type Key = mutable.WrappedArray[Byte] - val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty + val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty private var headersCollection: SortedMap[Int, List[ModifierId]] = SortedMap.empty[Int, List[ModifierId]] private var isChainSynced = false @@ -36,18 +36,23 @@ object ModifiersCache extends StrictLogging { value match { case header: Header => val possibleHeadersAtCurrentHeight: List[ModifierId] = headersCollection.getOrElse(header.height, List()) - logger.debug(s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}") + logger.debug( + s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) val updatedHeadersAtCurrentHeight: List[ModifierId] = header.id :: possibleHeadersAtCurrentHeight - logger.debug(s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}") + logger.debug( + s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) headersCollection = headersCollection.updated(header.height, updatedHeadersAtCurrentHeight) case _ => } - if (size > history.settings.node.modifiersCacheSize) cache.find { case (_, modifier) => - history.testApplicable(modifier) match { - case Right(_) | Left(_: NonFatalValidationError) => false - case _ => true - } + if (size > history.settings.node.modifiersCacheSize) cache.find { + case (_, modifier) => + history.testApplicable(modifier) match { + case Right(_) | Left(_: NonFatalValidationError) => false + case _ => true + } }.map(mod => remove(mod._1)) } @@ -64,13 +69,19 @@ object ModifiersCache extends StrictLogging { def findCandidateKey(history: History): List[Key] = { - def isApplicable(key: Key): Boolean = cache.get(key).exists(modifier => history.testApplicable(modifier) match { - case Left(_: FatalValidationError) => remove(key); false - case Right(_) => true - case Left(_) => false - }) + def isApplicable(key: Key): Boolean = + cache + .get(key) + .exists( + modifier => + history.testApplicable(modifier) match { + case Left(_: FatalValidationError) => remove(key); false + case Right(_) => true + case Left(_) => false + } + ) - def getHeadersKeysAtHeight(height: Int): List[Key] = { + def getHeadersKeysAtHeight(height: Int): List[Key] = headersCollection.get(height) match { case Some(headersIds) => headersIds.map(new mutable.WrappedArray.ofByte(_)).collect { @@ -79,26 +90,28 @@ object ModifiersCache extends StrictLogging { case None => List.empty[Key] } - } def findApplicablePayloadAtHeight(height: Int): List[Key] = { history.headerIdsAtHeight(height).view.flatMap(history.getHeaderById).collect { case header: Header if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => new mutable.WrappedArray.ofByte(header.payloadId) } - }.toList - - def exhaustiveSearch: List[Key] = List(cache.find { case (k, v) => - v match { - case _: Header if history.getBestHeaderId.exists(headerId => headerId sameElements v.parentId) => true - case _ => - val isApplicableMod: Boolean = isApplicable(k) - isApplicableMod - } - }).collect { case Some(v) => v._1 } + }.toList + + def exhaustiveSearch: List[Key] = + List(cache.find { + case (k, v) => + v match { + case _: Header if history.getBestHeaderId.exists(headerId => headerId sameElements v.parentId) => true + case _ => + val isApplicableMod: Boolean = isApplicable(k) + isApplicableMod + } + }).collect { case Some(v) => v._1 } @tailrec - def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, prevKeys: List[Key] = List.empty[Key]): List[Key] = { + def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, + prevKeys: List[Key] = List.empty[Key]): List[Key] = { val payloads = findApplicablePayloadAtHeight(atHeight) if (payloads.nonEmpty) applicableBestPayloadChain(atHeight + 1, prevKeys ++ payloads) else prevKeys @@ -112,9 +125,9 @@ object ModifiersCache extends StrictLogging { logger.debug(s"Drop height ${history.getBestHeaderHeight + 1} in HeadersCollection") val res = value.map(cache.get(_)).collect { case Some(v: Header) - if (history.getBestHeaderHeight == history.settings.constants.PreGenesisHeight && - (v.parentId sameElements Header.GenesisParentId) || - history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => + if (history.getBestHeaderHeight == history.settings.constants.PreGenesisHeight && + (v.parentId sameElements Header.GenesisParentId) || + history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => logger.debug(s"Find new bestHeader in cache: ${Algos.encode(v.id)}") new mutable.WrappedArray.ofByte(v.id) } @@ -124,26 +137,31 @@ object ModifiersCache extends StrictLogging { logger.debug(s"${history.getBestHeader}") logger.debug(s"${history.getBestHeaderHeight}") logger.debug(s"${headersCollection.get(history.getBestHeaderHeight + 1).map(_.map(Algos.encode))}") - logger.debug(s"No header in cache at height ${history.getBestHeaderHeight + 1}. " + - s"Trying to find in range [${history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]") - (history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth to history.getBestHeaderHeight).flatMap(height => - getHeadersKeysAtHeight(height) - ).toList + logger.debug( + s"No header in cache at height ${history.getBestHeaderHeight + 1}. " + + s"Trying to find in range [${history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]" + ) + (history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth to history.getBestHeaderHeight) + .flatMap(height => getHeadersKeysAtHeight(height)) + .toList } } if (bestHeadersIds.nonEmpty) bestHeadersIds - else history.headerIdsAtHeight(history.getBestBlockHeight + 1).headOption match { - case Some(id) => history.getHeaderById(id) match { - case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => - List(new mutable.WrappedArray.ofByte(header.payloadId)) - case _ if history.isFullChainSynced => exhaustiveSearch - case _ => List.empty[Key] + else + history.headerIdsAtHeight(history.getBestBlockHeight + 1).headOption match { + case Some(id) => + history.getHeaderById(id) match { + case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => + List(new mutable.WrappedArray.ofByte(header.payloadId)) + case _ if history.isFullChainSynced => exhaustiveSearch + case _ => List.empty[Key] + } + case None if isChainSynced => + logger.debug(s"No payloads for current history") + exhaustiveSearch + case None => + logger.debug(s"No payloads for current history") + List.empty[Key] } - case None if isChainSynced => - logger.debug(s"No payloads for current history") - exhaustiveSearch - case None => logger.debug(s"No payloads for current history") - List.empty[Key] - } } -} \ No newline at end of file +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index bfed745d26..8d6c367294 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -14,26 +14,9 @@ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ - NodeView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader, - UpdateInformation -} -import encry.nvg.SnapshotProcessor.{ - FastSyncDone, - FastSyncFinished, - HeaderChainIsSynced, - RemoveRedundantManifestIds, - RequiredManifestHeightAndId, - SnapshotChunk, - TreeChunks -} +import encry.nvg.NodeViewHolder._ import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -48,9 +31,9 @@ import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } From d40d1af2eed1fb589078f0f30375ae2d087a4505 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 15:14:35 +0300 Subject: [PATCH 059/119] change GetPeerWith(Older/Equal/Younger)History --- .../scala/encry/network/MessageBuilder.scala | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 941cb03de1..801c8b4441 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -9,7 +9,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} @@ -40,7 +40,7 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].foreach { peer => (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) @@ -86,25 +86,29 @@ object MessageBuilder { case class GetPeerInfo(peerIp: InetSocketAddress) trait GetPeerByPredicate { + def predicate: PeerInfo => Boolean - def ||(that: GetPeerByPredicate): GetPeerByPredicate = new GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = info => this.predicate(info) || that.predicate(info) + + def ||(that: GetPeerByPredicate): GetPeerByPredicate = { + println("invoke!") + GetPeerByPredicate((info: PeerInfo) => predicate(info) || that.predicate(info)) } - def &&(that: GetPeerByPredicate): GetPeerByPredicate =new GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = info => this.predicate(info) && that.predicate(info) + def &&(that: GetPeerByPredicate): GetPeerByPredicate = { + println("invoke!") + val newPredicate = (info: PeerInfo) => this.predicate.andThen(res => that.predicate(info) && res)(info) + GetPeerByPredicate(newPredicate) } } - final case class GetPeerWithEqualHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Equal - } - final case class GetPeerWithOlderHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Older + object GetPeerByPredicate { + def apply(peerPredicate: PeerInfo => Boolean): GetPeerByPredicate = new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = peerPredicate + } } - final case class GetPeerWithYoungerHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger - } + val PeerWithEqualHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Equal) + val PeerWithOlderHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Older) + val PeerWithYoungerHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Younger) def props(peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) From 9ce0eb9d29100c8e7743b274718d326fe50d95d5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 15:52:15 +0300 Subject: [PATCH 060/119] fix payload downloading --- src/main/scala/encry/network/MessageBuilder.scala | 9 ++++----- src/main/scala/encry/network/PK.scala | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 801c8b4441..544af268ea 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -29,7 +29,7 @@ case class MessageBuilder(peersKeeper: ActorRef, case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => - logger.info(s"Going to req mods from ${peer.socketAddress}") + logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) @@ -40,15 +40,16 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].map { peer => + logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } - context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) @@ -90,11 +91,9 @@ object MessageBuilder { def predicate: PeerInfo => Boolean def ||(that: GetPeerByPredicate): GetPeerByPredicate = { - println("invoke!") GetPeerByPredicate((info: PeerInfo) => predicate(info) || that.predicate(info)) } def &&(that: GetPeerByPredicate): GetPeerByPredicate = { - println("invoke!") val newPredicate = (info: PeerInfo) => this.predicate.andThen(res => that.predicate(info) && res)(info) GetPeerByPredicate(newPredicate) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 8a87c7cb19..0ab62b5224 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -125,7 +125,7 @@ class PK(networkSettings: NetworkSettings, case predicate: GetPeerByPredicate => connectedPeers.getAll.find { case (_, info) => predicate.predicate(info) }.map { - case (_, info) => sender() ! info.connectedPeer.handlerRef + case (_, info) => sender() ! info.connectedPeer } case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { From e3d03f0fbfec48f3ddf247dcfebce42ee7fdc2c8 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 16:22:12 +0300 Subject: [PATCH 061/119] start pipeline of transactions downloading --- src/main/scala/encry/network/NetworkRouter.scala | 15 ++++++++++----- .../encry/view/mempool/IntermediaryMempool.scala | 16 +++++++++++----- .../encry/view/mempool/MemoryPoolProcessor.scala | 3 --- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index d1b5968234..be370e7aae 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -9,9 +9,9 @@ import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} +import encry.network.MessageBuilder.MsgSent import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} @@ -19,6 +19,7 @@ import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, N import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -33,6 +34,7 @@ class NetworkRouter(settings: NetworkSettings, var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty var handlerForMods: ActorRef = ActorRef.noSender + var txsHandler: ActorRef = ActorRef.noSender IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) @@ -45,9 +47,10 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) messagesHandlers += (ids -> handler) - case msg: MsgSent => context.stop(sender()) + case _: MsgSent => context.stop(sender()) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") case RegisterForModsHandling => handlerForMods = sender() + case RegisterForTxHandling => txsHandler = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") } @@ -62,13 +65,14 @@ class NetworkRouter(settings: NetworkSettings, def businessLogic: Receive = { case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => - logger.debug(s"Got ${message.messageName} on the NetworkController.") + logger.debug(s"Got ${message.messageName} on the NetworkRouter.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: SemanticallySuccessfulModifier => deliveryManager ! msg - case msg: ModifierFromNetwork => handlerForMods ! msg + case msg: ModifierFromNetwork if msg.modTypeId != Transaction.modifierTypeId => handlerForMods ! msg + case msg: ModifierFromNetwork => txsHandler ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => context.actorOf( @@ -127,6 +131,7 @@ object NetworkRouter { modBytes: Array[Byte]) case object RegisterForModsHandling + case object RegisterForTxHandling def props(settings: NetworkSettings, blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index e798dbd897..62b41dc8bb 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -4,20 +4,21 @@ import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.MemoryPoolProcessor.RequestedModifiersForRemote import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, minerReference: ActorRef, - influxReference: Option[ActorRef]) + influxReference: Option[ActorRef], + networkRouter: ActorRef) extends Actor with StrictLogging { @@ -29,6 +30,10 @@ class IntermediaryMempool(settings: EncryAppSettings, val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") + override def preStart(): Unit = { + networkRouter ! RegisterForTxHandling + } + override def receive(): Receive = { case msg @ InvalidModifier(_) => // to nvsh case msg @ BanPeer(_, _) => // to peersKeeper @@ -37,8 +42,8 @@ class IntermediaryMempool(settings: EncryAppSettings, case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor - case msg @ RequestedModifiersForRemote(_, _) => // to network case msg @ RequestFromLocal(_, _, _) => // to network + case msg @ ModifierFromNetwork(_, _, _, _) => txValidator ! msg case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor case msg @ IsChainSynced(_) => mempoolProcessor ! msg } @@ -48,8 +53,9 @@ object IntermediaryMempool { def props(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, minerReference: ActorRef, - influxReference: Option[ActorRef]) = - Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference)) + influxReference: Option[ActorRef], + networkRouter: ActorRef) = + Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 0c919244b5..4736009fbd 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -104,7 +104,4 @@ object MemoryPoolProcessor { def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter - - final case class RequestedModifiersForRemote(peer: InetSocketAddress, txs: Seq[Transaction]) - } From b493b1715408b228134a705cb2d199fa0a9e83b2 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 16:25:44 +0300 Subject: [PATCH 062/119] code style updated --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../encry/api/http/DataHolderForApi.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 2 +- src/main/scala/encry/network/DM.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 3 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 35 +-- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +- .../{ => fast/sync}/SnapshotDownloader.scala | 26 +- .../sync}/SnapshotIntermediary.scala | 2 +- .../{ => fast/sync}/SnapshotProcessor.scala | 16 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 5 + .../encry/nvg/{ => nvhg}/NodeViewHolder.scala | 27 +- .../nvg/nvhg/NodeViewHolderController.scala | 5 + .../scala/encry/nvg/nvhg/StateApplier.scala | 5 + .../sync/SnapshotDownloadController.scala | 4 +- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- .../encry/view/fast/sync/SnapshotHolder.scala | 27 +- .../sync/SnapshotProcessorStorageAPI.scala | 2 +- .../scala/encry/view/mempool/MemoryPool.scala | 2 +- .../view/mempool/MemoryPoolProcessor.scala | 3 +- .../encry/view/state/avlTree/AvlTree.scala | 254 +++++++++--------- 23 files changed, 225 insertions(+), 223 deletions(-) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotDownloader.scala (90%) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotIntermediary.scala (82%) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotProcessor.scala (96%) create mode 100644 src/main/scala/encry/nvg/nvhg/HistoryApplier.scala rename src/main/scala/encry/nvg/{ => nvhg}/NodeViewHolder.scala (97%) create mode 100644 src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala create mode 100644 src/main/scala/encry/nvg/nvhg/StateApplier.scala diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index bcc7d55023..c46e2d7749 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,7 +7,7 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} -import encry.nvg.SnapshotProcessor +import encry.nvg.fast.sync.SnapshotProcessor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 935b592d8d..f29e856311 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,7 +21,7 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.NodeViewHolder.NodeViewChange +import encry.nvg.nvhg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 660cb5324e..c61d078e4a 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -14,7 +14,7 @@ import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 889a14f7b0..5ab898a9c8 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -14,7 +14,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import encry.nvg.nvhg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable import scala.concurrent.Future diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index be370e7aae..bb4ad9ecfd 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,6 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -17,7 +18,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index e3d4b4bbbb..9313e51524 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -33,7 +33,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.nvhg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import scala.util.Try diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 92a7054b5a..c2de5291eb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,41 +1,28 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.{ - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} -import encry.nvg.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } +import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.fast.sync.SnapshotProcessor +import encry.nvg.nvhg.NodeViewHolder.{RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.nvg.nvhg.NodeViewHolder import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e3990ec576..e63f0eb669 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } -import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +import encry.nvg.nvhg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8415387d83..75d88241b7 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,26 +1,22 @@ package encry.nvg -import akka.actor.{ Actor, Cancellable, Props } +import akka.actor.{Actor, Cancellable, Props} import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } +import encry.nvg.nvhg.NodeViewHolder +import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, UpdateHistoryReader} import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - ModifiersNetworkMessage, - RequestModifiersNetworkMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala similarity index 90% rename from src/main/scala/encry/nvg/SnapshotDownloader.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala index 902bba67c3..bcc4ed26c5 100644 --- a/src/main/scala/encry/nvg/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala @@ -1,26 +1,14 @@ -package encry.nvg +package encry.nvg.fast.sync -import akka.actor.{ Actor, Cancellable } -import cats.syntax.either._ -import cats.syntax.option._ +import akka.actor.{Actor, Cancellable} import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{ InvalidChunkMessage, InvalidStateAfterFastSync } import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.PeersKeeper.BanPeer -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{BroadcastManifestRequestMessage, RequiredManifestHeightAndId} +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings -import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } -import encry.view.fast.sync.{ SnapshotDownloadController, SnapshotHolder } -import encry.view.history.{ History, HistoryReader } -import encry.view.wallet.EncryWallet -import org.encryfoundation.common.modifiers.history.{ Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - RequestChunkMessage, - ResponseChunkMessage, - ResponseManifestMessage -} +import encry.view.fast.sync.{SnapshotDownloadController, SnapshotHolder} +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.Algos class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/nvg/SnapshotIntermediary.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala similarity index 82% rename from src/main/scala/encry/nvg/SnapshotIntermediary.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala index 9e08c81b36..bf8c04b28e 100644 --- a/src/main/scala/encry/nvg/SnapshotIntermediary.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala @@ -1,4 +1,4 @@ -package encry.nvg +package encry.nvg.fast.sync import akka.actor.Actor diff --git a/src/main/scala/encry/nvg/SnapshotProcessor.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala similarity index 96% rename from src/main/scala/encry/nvg/SnapshotProcessor.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala index ed002d7f03..945c6cb544 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala @@ -1,23 +1,19 @@ -package encry.nvg +package encry.nvg.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import akka.actor.{Actor, ActorRef, Cancellable, Props} -import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} -import encry.network.Broadcast -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.DeliveryManager.CheckDelivery +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.{BanPeer, SendToNetwork} -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{DropProcessedCount, HeaderChainIsSynced, RemoveRedundantManifestIds, TreeChunks} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala new file mode 100644 index 0000000000..3f47124fd2 --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class HistoryApplier { + +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala similarity index 97% rename from src/main/scala/encry/nvg/NodeViewHolder.scala rename to src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 8d6c367294..3dd5f03522 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -1,4 +1,4 @@ -package encry.nvg +package encry.nvg.nvhg import java.io.File @@ -12,11 +12,28 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder._ -import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + FastSyncFinished, + HeaderChainIsSynced, + RemoveRedundantManifestIds, + SnapshotChunk, + TreeChunks +} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.nvhg.NodeViewHolder.{ + NodeView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader, + UpdateInformation +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala new file mode 100644 index 0000000000..f64e04a22f --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class NodeViewHolderController { + +} diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala new file mode 100644 index 0000000000..5316a2b46a --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/StateApplier.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class StateApplier { + +} diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index d2635d4f21..45ec35fc51 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -8,8 +8,8 @@ import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId -import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index 79d3b4d1df..7bee1e8eaa 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,7 +1,7 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index ef047908a7..adbb1112b1 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -7,36 +7,33 @@ import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.{ - SnapshotChunk, - SnapshotChunkSerializer, - SnapshotManifest, - SnapshotManifestSerializer -} -import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.fast.sync.SnapshotProcessor.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings -import encry.storage.{ RootNodesStorage, VersionalStorage } -import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } +import encry.storage.{RootNodesStorage, VersionalStorage} +import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper -import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.view.fast.sync.FastSyncExceptions._ import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree._ import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.wallet.EncryWallet -import io.iohk.iodb.{ ByteArrayWrapper, LSMStore } +import io.iohk.iodb.{ByteArrayWrapper, LSMStore} import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } -import org.iq80.leveldb.{ DB, Options } +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.iq80.leveldb.{DB, Options} import scorex.utils.Random -import scala.collection.immutable.{ HashMap, HashSet } +import scala.collection.immutable.{HashMap, HashSet} import scala.language.postfixOps -import scala.util.{ Failure, Success } +import scala.util.{Failure, Success} final case class SnapshotHolder( settings: EncryAppSettings, diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index f52505f78c..26b7e0ca63 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,7 +3,7 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } +import encry.nvg.fast.sync.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 695ded5134..0c4ae8dfec 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -5,7 +5,7 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 4736009fbd..23a0fbbc00 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -12,7 +12,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter, RequestedModifiersForRemote} +import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -55,7 +55,6 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) .map(Algos.encode) .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } .flatten - context.parent ! RequestedModifiersForRemote(remote, modifiersIds) logger.debug( s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 9b97e83d45..9ab46c63ef 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,8 +4,8 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotChunk -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} import encry.view.state.UtxoState @@ -18,10 +18,12 @@ import org.encryfoundation.common.utils.TaggedTypes.Height import scala.util.Try -final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], - avlStorage: VersionalStorage, - rootNodesStorage: RootNodesStorage[K, V], - saveRootNodes: Boolean = false) extends AutoCloseable with StrictLogging { +final case class AvlTree[K: Hashable: Order, V](rootNode: Node[K, V], + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V], + saveRootNodes: Boolean = false) + extends AutoCloseable + with StrictLogging { implicit def nodeOrder(implicit ord: Order[K]): Order[Node[K, V]] = new Order[Node[K, V]] { override def compare(x: Node[K, V], y: Node[K, V]): Int = ord.compare(x.key, y.key) @@ -31,31 +33,29 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], val rootHash: Array[Byte] = rootNode.hash - def insertAndDeleteMany(version: StorageVersion, - toInsert: List[(K, V)], - toDelete: List[K], - stateHeight: Height = Height @@ 0, - saveRootNodesFlag: Boolean = false) - (implicit kSer: Serializer[K], - vSer: Serializer[V], - kM: Monoid[K], - vM: Monoid[V]): AvlTree[K, V] = { + def insertAndDeleteMany( + version: StorageVersion, + toInsert: List[(K, V)], + toDelete: List[K], + stateHeight: Height = Height @@ 0, + saveRootNodesFlag: Boolean = false + )(implicit kSer: Serializer[K], vSer: Serializer[V], kM: Monoid[K], vM: Monoid[V]): AvlTree[K, V] = { val deleteStartTime = System.nanoTime() val rootAfterDelete = toDelete.foldLeft(rootNode) { case (prevRoot, toDeleteKey) => deleteKey(toDeleteKey, prevRoot) } val avlDeleteTime = System.nanoTime() - deleteStartTime - logger.debug(s"avlDeleteTime: ${avlDeleteTime/1000000L} ms") + logger.debug(s"avlDeleteTime: ${avlDeleteTime / 1000000L} ms") val insertStartTime = System.nanoTime() val newRoot = toInsert.foldLeft(rootAfterDelete) { case (prevRoot, (keyToInsert, valueToInsert)) => insert(keyToInsert, valueToInsert, prevRoot) } val insertTime = System.nanoTime() - insertStartTime - logger.debug(s"avlInsertTime: ${insertTime/1000000L} ms") + logger.debug(s"avlInsertTime: ${insertTime / 1000000L} ms") val startPackingTime = System.nanoTime() - logger.debug(s"Packing time: ${(System.nanoTime() - startPackingTime)/1000000} ms") + logger.debug(s"Packing time: ${(System.nanoTime() - startPackingTime) / 1000000} ms") val startInsertTime = System.nanoTime() logger.debug(s"Insert in avl version ${Algos.encode(version)}") avlStorage.insert( @@ -64,18 +64,19 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case (key, value) => StorageKey @@ AvlTree.elementKey(kSer.toBytes(key)) -> StorageValue @@ vSer.toBytes(value) } ++ - List(AvlTree.rootNodeKey -> StorageValue @@ newRoot.hash, - UtxoState.bestHeightKey -> StorageValue @@ Ints.toByteArray(stateHeight)), - toDelete.map(key => - StorageKey @@ AvlTree.elementKey(kSer.toBytes(key)) - ) + List(AvlTree.rootNodeKey -> StorageValue @@ newRoot.hash, + UtxoState.bestHeightKey -> StorageValue @@ Ints.toByteArray(stateHeight)), + toDelete.map(key => StorageKey @@ AvlTree.elementKey(kSer.toBytes(key))) ) - logger.debug(s"Insertion time: ${(System.nanoTime() - startInsertTime)/1000000L} ms") - val newRootNodesStorage = if (saveRootNodesFlag || saveRootNodes) rootNodesStorage.insert( - version, - newRoot, - stateHeight - ) else rootNodesStorage + logger.debug(s"Insertion time: ${(System.nanoTime() - startInsertTime) / 1000000L} ms") + val newRootNodesStorage = + if (saveRootNodesFlag || saveRootNodes) + rootNodesStorage.insert( + version, + newRoot, + stateHeight + ) + else rootNodesStorage AvlTree(newRoot, avlStorage, newRootNodesStorage, saveRootNodesFlag || saveRootNodes) } @@ -95,10 +96,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } def findKey(key: K, node: Node[K, V]): Option[V] = node match { - case internalNode: InternalNode[K, V] if key > internalNode.key=> findKey(key, internalNode.rightChild) - case internalNode: InternalNode[K, V] if key < internalNode.key=> findKey(key, internalNode.leftChild) - case n: Node[K, V] if n.key === key => Some(n.value) - case _ => None + case internalNode: InternalNode[K, V] if key > internalNode.key => findKey(key, internalNode.rightChild) + case internalNode: InternalNode[K, V] if key < internalNode.key => findKey(key, internalNode.leftChild) + case n: Node[K, V] if n.key === key => Some(n.value) + case _ => None } def get(k: K)(implicit kSer: Serializer[K], vSer: Serializer[V]): Option[V] = @@ -107,10 +108,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], def contains(k: K)(implicit kSer: Serializer[K]): Boolean = avlStorage.get(StorageKey !@@ AvlTree.elementKey(kSer.toBytes(k))).isDefined - def deleteKey(key: K, node: Node[K, V])(implicit m: Monoid[K], - v: Monoid[V], - kSer: Serializer[K], - vSer: Serializer[V]): Node[K, V] = delete(node, key) + def deleteKey( + key: K, + node: Node[K, V] + )(implicit m: Monoid[K], v: Monoid[V], kSer: Serializer[K], vSer: Serializer[V]): Node[K, V] = delete(node, key) private def delete(node: Node[K, V], key: K)( implicit m: Monoid[K], @@ -132,7 +133,7 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], balance(childUpdated) } else if (internalNode.key < key) { val newRightChild = delete(internalNode.rightChild, key) - val childUpdated = internalNode.updateChilds(newRightChild = newRightChild) + val childUpdated = internalNode.updateChilds(newRightChild = newRightChild) balance(childUpdated) } else { val theClosestValue = findTheClosestValue(internalNode, internalNode.key) @@ -188,8 +189,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } - def rollbackTo(to: StorageVersion, additionalBlocks: List[Block]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + def rollbackTo( + to: StorageVersion, + additionalBlocks: List[Block] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = Try { logger.info(s"Rollback avl to version: ${Algos.encode(to)}") logger.info(s"Versions in storage: ${avlStorage.versions.map(Algos.encode).mkString(",")}") @@ -198,18 +201,21 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], avlStorage.rollbackTo(to) logger.info(s"Storage success rolled back") logger.info(s"rootNodeKey: ${Algos.encode(avlStorage.get(AvlTree.rootNodeKey).getOrElse(Array.emptyByteArray))}") - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) logger.info(s"root node hash after rollback: ${Algos.encode(newRoot.hash)}") AvlTree[K, V](newRoot, avlStorage, newStorage) } - def restore(additionalBlocks: List[Block]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = - Try { - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) - logger.info(s"root node hash after restore: ${Algos.encode(newRoot.hash)}") - AvlTree[K, V](newRoot, avlStorage, newStorage) - } + def restore( + additionalBlocks: List[Block] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + Try { + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + logger.info(s"root node hash after restore: ${Algos.encode(newRoot.hash)}") + AvlTree[K, V](newRoot, avlStorage, newStorage) + } private def getRightPath(node: Node[K, V]): List[Node[K, V]] = node match { case shadowNode: ShadowNode[K, V] => @@ -231,39 +237,39 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case _: EmptyNode[K, V] => List.empty } - private def insert(newKey: K, newValue: V, node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = node match { - case shadowNode: ShadowNode[K, V] => - val restoredNode = shadowNode.restoreFullNode(avlStorage) - insert(newKey, newValue, restoredNode) - case _: EmptyNode[K, V] => LeafNode[K, V](newKey, newValue) - case leafNode: LeafNode[K, V] => - if (leafNode.key === newKey) leafNode.copy(value = newValue) - else { - val newInternalNode = InternalNode[K, V](leafNode.key, leafNode.value, height = 1, balance = 0) - insert( - newKey, - newValue, - newInternalNode - ) - } - case internalNode: InternalNode[K, V] => - if (internalNode.key > newKey) { - val newLeftChild = insert(newKey, newValue, internalNode.leftChild) - val newNode = internalNode.updateChilds(newLeftChild = newLeftChild) - balance(newNode) - } else { - val newRightChild = insert(newKey, newValue, internalNode.rightChild) - val newNode = internalNode.updateChilds(newRightChild = newRightChild) - balance(newNode) - } - } + private def insert(newKey: K, newValue: V, node: Node[K, V])(implicit kMonoid: Monoid[K], + kSer: Serializer[K], + vMonoid: Monoid[V], + vSer: Serializer[V]): Node[K, V] = node match { + case shadowNode: ShadowNode[K, V] => + val restoredNode = shadowNode.restoreFullNode(avlStorage) + insert(newKey, newValue, restoredNode) + case _: EmptyNode[K, V] => LeafNode[K, V](newKey, newValue) + case leafNode: LeafNode[K, V] => + if (leafNode.key === newKey) leafNode.copy(value = newValue) + else { + val newInternalNode = InternalNode[K, V](leafNode.key, leafNode.value, height = 1, balance = 0) + insert( + newKey, + newValue, + newInternalNode + ) + } + case internalNode: InternalNode[K, V] => + if (internalNode.key > newKey) { + val newLeftChild = insert(newKey, newValue, internalNode.leftChild) + val newNode = internalNode.updateChilds(newLeftChild = newLeftChild) + balance(newNode) + } else { + val newRightChild = insert(newKey, newValue, internalNode.rightChild) + val newNode = internalNode.updateChilds(newRightChild = newRightChild) + balance(newNode) + } + } - private def balance(node: Node[K, V]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { + private def balance( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -310,11 +316,9 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case _ => -1 } - private def rightRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def rightRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -331,16 +335,14 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } val newLeftChildForPrevRoot = newRoot.rightChild.selfInspection - val prevRoot = internalNode.updateChilds(newLeftChild = newLeftChildForPrevRoot) + val prevRoot = internalNode.updateChilds(newLeftChild = newLeftChildForPrevRoot) newRoot.updateChilds(newRightChild = prevRoot) } }.selfInspection - private def leftRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def leftRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -357,16 +359,14 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } val newRightChildForPrevRoot = newRoot.leftChild.selfInspection - val prevRoot = internalNode.updateChilds(newRightChild = newRightChildForPrevRoot) + val prevRoot = internalNode.updateChilds(newRightChild = newRightChildForPrevRoot) newRoot.updateChilds(newLeftChild = prevRoot) } }.selfInspection - private def rlRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def rlRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -380,11 +380,9 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } }.selfInspection - private def lrRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def lrRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -416,8 +414,12 @@ object AvlTree extends StrictLogging { AvlTree(rootNode, avlStorage, rootNodesStorage) } - def rollbackTo[K: Hashable: Order, V](to: StorageVersion, additionalBlocks: List[Block], avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + def rollbackTo[K: Hashable: Order, V]( + to: StorageVersion, + additionalBlocks: List[Block], + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = Try { logger.info(s"Rollback avl to version: ${Algos.encode(to)}") logger.info(s"Versions in storage: ${avlStorage.versions.map(Algos.encode).mkString(",")}") @@ -425,7 +427,8 @@ object AvlTree extends StrictLogging { avlStorage.rollbackTo(to) logger.info(s"Storage success rolled back") logger.info(s"rootNodeKey: ${Algos.encode(avlStorage.get(AvlTree.rootNodeKey).getOrElse(Array.emptyByteArray))}") - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) logger.info(s"root node hash after rollback: ${Algos.encode(newRoot.hash)}") AvlTree[K, V](newRoot, avlStorage, newStorage) } @@ -439,28 +442,30 @@ object AvlTree extends StrictLogging { case object EMPTY extends Direction } - def apply[K: Monoid: Order: Hashable : Serializer, - V: Monoid : Serializer](avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V]): AvlTree[K, V] = - { - rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) - new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) - } + def apply[K: Monoid: Order: Hashable: Serializer, V: Monoid: Serializer]( + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V] + ): AvlTree[K, V] = { + rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) + new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) + } def elementKey(key: Array[Byte]): Array[Byte] = (0: Byte) +: key def nodeKey(key: Array[Byte]): Array[Byte] = (1: Byte) +: key - def getChunks(node: Node[StorageKey, StorageValue], - currentChunkHeight: Int, - avlStorage: VersionalStorage) - (implicit kSer: Serializer[StorageKey], - vSer: Serializer[StorageValue], - kM: Monoid[StorageKey], - vM: Monoid[StorageValue], - hashKey: Hashable[StorageKey]): List[SnapshotChunk] = { - - def restoreNodesUntilDepthAndReturnLeafs(depth: Int, - node: Node[StorageKey, StorageValue]): (Node[StorageKey, StorageValue], List[Node[StorageKey, StorageValue]]) = node match { + def getChunks(node: Node[StorageKey, StorageValue], currentChunkHeight: Int, avlStorage: VersionalStorage)( + implicit kSer: Serializer[StorageKey], + vSer: Serializer[StorageValue], + kM: Monoid[StorageKey], + vM: Monoid[StorageValue], + hashKey: Hashable[StorageKey] + ): List[SnapshotChunk] = { + + def restoreNodesUntilDepthAndReturnLeafs( + depth: Int, + node: Node[StorageKey, StorageValue] + ): (Node[StorageKey, StorageValue], List[Node[StorageKey, StorageValue]]) = node match { case shadowNode: ShadowNode[StorageKey, StorageValue] => val newNode = shadowNode.restoreFullNode(avlStorage) restoreNodesUntilDepthAndReturnLeafs(depth, newNode) @@ -475,11 +480,12 @@ object AvlTree extends StrictLogging { ) -> (rightSubTreeChildren ++ leftSubTreeChildren) case internalNode: InternalNode[StorageKey, StorageValue] => internalNode -> List(internalNode.leftChild, internalNode.rightChild) - case leaf: LeafNode[StorageKey, StorageValue] => leaf -> List.empty[Node[StorageKey, StorageValue]] + case leaf: LeafNode[StorageKey, StorageValue] => leaf -> List.empty[Node[StorageKey, StorageValue]] case emptyNode: EmptyNode[StorageKey, StorageValue] => emptyNode -> List.empty } - val (rootChunk: Node[StorageKey, StorageValue], rootChunkChildren) = restoreNodesUntilDepthAndReturnLeafs(currentChunkHeight, node) + val (rootChunk: Node[StorageKey, StorageValue], rootChunkChildren) = + restoreNodesUntilDepthAndReturnLeafs(currentChunkHeight, node) SnapshotChunk(rootChunk, ChunkId @@ rootChunk.hash) :: rootChunkChildren.flatMap(node => getChunks(node, currentChunkHeight, avlStorage)) } From d92a85c466eb4f9be41cd1c21ec82f305f9bf7d9 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 17:41:53 +0300 Subject: [PATCH 063/119] logs improved --- src/main/scala/encry/network/DM.scala | 4 ++-- src/main/scala/encry/nvg/ModifiersValidator.scala | 2 ++ src/main/scala/encry/view/state/UtxoState.scala | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 5ab898a9c8..72804c6a5c 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -42,8 +42,8 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) - context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! - AwaitingRequest(peer, modTypeId, modId, 1) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( + self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e63f0eb669..5dfa628e8b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -55,6 +55,8 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } } + + private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 614098a537..ec9d901776 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -76,7 +76,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], case block: Block => logger.info(s"\n\nStarting to applyModifier as a Block: ${Algos.encode(mod.id)} to state at height ${block.header.height}") logger.info(s"State root should be: ${Algos.encode(block.header.stateRoot)}") - logger.info(s"Current root node hash: ${tree.rootNode.hash}") + logger.info(s"Current root node hash: ${Algos.encode(tree.rootNode.hash)}") val lastTxId = block.payload.txs.last.id val totalFees: Amount = block.payload.txs.init.map(_.fee).sum val validstartTime = System.nanoTime() @@ -120,7 +120,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], s"State root should be ${Algos.encode(block.header.stateRoot)} but got " + s"${Algos.encode(newTree.rootNode.hash)}")).asLeft[UtxoState] } else { - logger.info(s"After applying root node: ${newTree.rootNode.hash}") + logger.info(s"After applying root node: ${Algos.encode(newTree.rootNode.hash)}") UtxoState( newTree, Height @@ block.header.height, @@ -139,7 +139,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], def rollbackTo(version: VersionTag, additionalBlocks: List[Block]): Try[UtxoState] = Try{ logger.info(s"Rollback utxo to version: ${Algos.encode(version)}") val rollbackedAvl = AvlTree.rollbackTo(StorageVersion !@@ version, additionalBlocks, tree.avlStorage, tree.rootNodesStorage).get - logger.info(s"UTXO -> rollbackTo ->${tree.avlStorage.get(UtxoState.bestHeightKey)} ") + logger.info(s"UTXO -> rollbackTo -> ${tree.avlStorage.get(UtxoState.bestHeightKey).map(Ints.fromByteArray)}.") val height: Height = Height !@@ Ints.fromByteArray(tree.avlStorage.get(UtxoState.bestHeightKey).get) UtxoState(rollbackedAvl, height, constants, influxRef) } @@ -147,7 +147,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], def restore(additionalBlocks: List[Block]): Try[UtxoState] = Try { logger.info(s"Rollback utxo from storage: ${Algos.encode(version)}") val rollbackedAvl = tree.restore(additionalBlocks).get - logger.info(s"UTXO -> rollbackTo ->${tree.avlStorage.get(UtxoState.bestHeightKey)} ") + logger.info(s"UTXO -> restore -> ${tree.avlStorage.get(UtxoState.bestHeightKey).map(Ints.fromByteArray)}.") val height: Height = Height !@@ Ints.fromByteArray(tree.avlStorage.get(UtxoState.bestHeightKey).get) UtxoState(rollbackedAvl, height, constants, influxRef) } From da1b14390ddf9648bcb0004e5e31f54656d164aa Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 18:19:30 +0300 Subject: [PATCH 064/119] code style updated --- src/main/scala/encry/Starter.scala | 15 ++++---- .../scala/encry/nvg/IntermediaryNVH.scala | 38 ++++++++++++++----- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 6afb9ca961..17c2813c50 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -2,10 +2,10 @@ package encry import java.net.InetSocketAddress -import akka.actor.{Actor, ActorRef} +import akka.actor.{ Actor, ActorRef } import akka.http.scaladsl.Http import cats.Functor -import cats.data.{NonEmptyChain, Validated} +import cats.data.{ NonEmptyChain, Validated } import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -16,21 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{StartListening, prompt} +import encry.cli.ConsoleListener.{ prompt, StartListening } import encry.local.miner.Miner import encry.local.miner.Miner.StartMining -import encry.network.{NetworkRouter, NodeViewSynchronizer} +import encry.network.NetworkRouter import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender -import encry.utils.{Mnemonic, NetworkTimeProvider} -import encry.view.NodeViewHolder +import encry.utils.{ Mnemonic, NetworkTimeProvider } import encry.view.mempool.MemoryPool import encry.view.wallet.AccountManager import scala.concurrent.Future import scala.io.StdIn -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, @@ -43,7 +42,7 @@ class Starter(settings: EncryAppSettings, var initHttpApiServer: Option[Future[Http.ServerBinding]] = none - val preview = + val preview: String = """ |XXXXXX XX XX XXXXX XXXXXX XX XX |XX XXXX XX XX XX XX XXXXXX diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c2de5291eb..5947d99e8a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,28 +1,48 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.nvg.nvhg.NodeViewHolder.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.nvg.nvhg.NodeViewHolder import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, + RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( From adbdff42e0a98e3f18ffe8136d401294e2e42264 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 18:30:08 +0300 Subject: [PATCH 065/119] add some pipelines to mempool --- src/main/scala/encry/Starter.scala | 19 ++++-- .../scala/encry/network/NetworkRouter.scala | 7 +- .../scala/encry/nvg/nvhg/NodeViewHolder.scala | 2 +- .../encry/storage/RootNodesStorage.scala | 22 +++++-- .../scala/encry/view/mempool/MemoryPool.scala | 65 +++++++++++++++++++ .../view/mempool/MemoryPoolProcessor.scala | 11 ++-- .../view/mempool/TransactionsValidator.scala | 35 +++++----- .../encry/view/state/avlTree/AvlTree.scala | 2 +- 8 files changed, 124 insertions(+), 39 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 17c2813c50..265514609c 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -25,6 +25,9 @@ import encry.settings._ import encry.stats.StatsSender import encry.utils.{ Mnemonic, NetworkTimeProvider } import encry.view.mempool.MemoryPool +import encry.utils.{Mnemonic, NetworkTimeProvider} +import encry.view.NodeViewHolder +import encry.view.mempool.{IntermediaryMempool, MemoryPool} import encry.view.wallet.AccountManager import scala.concurrent.Future @@ -408,13 +411,13 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") - lazy val miner: ActorRef = + val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, influxRef, newSettings), "miner") - lazy val memoryPool: ActorRef = context.system.actorOf( - MemoryPool - .props(newSettings, timeProvider, miner, influxRef) - .withDispatcher("mempool-dispatcher") - ) +// lazy val memoryPool: ActorRef = context.system.actorOf( +// MemoryPool +// .props(newSettings, timeProvider, miner, influxRef) +// .withDispatcher("mempool-dispatcher") +// ) // val nodeViewHolder: ActorRef = context.system.actorOf( // NodeViewHolder // .props(memoryPool, influxRef, dataHolderForApi, newSettings) @@ -442,6 +445,10 @@ class Starter(settings: EncryAppSettings, IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) ) + val memoryPool = context.system.actorOf( + IntermediaryMempool.props(newSettings, timeProvider, miner, influxRef, networkRouter) + ) + if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index bb4ad9ecfd..a98a3393f5 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,7 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -21,7 +21,7 @@ import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConn import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ @@ -65,6 +65,9 @@ class NetworkRouter(settings: NetworkSettings, } def businessLogic: Receive = { + case mfn@MessageFromNetwork(inv@InvNetworkMessage(data), Some(_)) if data._1 == Transaction.modifierTypeId && inv.isValid(settings.syncPacketLength) => + logger.debug(s"Got ${inv.messageName} on the NetworkRouter.") + txsHandler ! mfn case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => logger.debug(s"Got ${message.messageName} on the NetworkRouter.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 3dd5f03522..6dcf1745e5 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -472,7 +472,7 @@ class NodeViewHolder( logger.info( s"State and history are inconsistent." + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers" + s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" ) val additionalBlocks = (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { diff --git a/src/main/scala/encry/storage/RootNodesStorage.scala b/src/main/scala/encry/storage/RootNodesStorage.scala index c47fccc0ad..66528341e1 100644 --- a/src/main/scala/encry/storage/RootNodesStorage.scala +++ b/src/main/scala/encry/storage/RootNodesStorage.scala @@ -1,17 +1,20 @@ package encry.storage -import java.io.{ BufferedOutputStream, File, FileOutputStream } -import java.nio.file.{ Files, Paths } -import cats.kernel.{ Monoid, Order } +import java.io.{BufferedOutputStream, File, FileOutputStream} +import java.nio.file.{Files, Paths} + +import cats.kernel.{Monoid, Order} import com.google.common.primitives.Ints +import com.typesafe.scalalogging.StrictLogging import encry.storage.VersionalStorage.StorageVersion -import encry.view.state.avlTree.utils.implicits.{ Hashable, Serializer } -import encry.view.state.avlTree.{ AvlTree, EmptyNode, Node, NodeSerilalizer } +import encry.view.state.avlTree.utils.implicits.{Hashable, Serializer} +import encry.view.state.avlTree.{AvlTree, EmptyNode, Node, NodeSerilalizer} import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.Height -import org.iq80.leveldb.{ DB, ReadOptions } +import org.iq80.leveldb.{DB, ReadOptions} import scorex.utils.Random + import scala.util.Try trait RootNodesStorage[K, V] extends AutoCloseable { @@ -29,7 +32,7 @@ object RootNodesStorage { def apply[K: Serializer: Monoid: Hashable: Order, V: Serializer: Monoid: Hashable](storage: DB, rollbackDepth: Int, - rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with AutoCloseable { + rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with AutoCloseable with StrictLogging { private def atHeightKey(height: Height): Array[Byte] = Ints.toByteArray(height) @@ -47,6 +50,8 @@ object RootNodesStorage { val bos = new BufferedOutputStream(new FileOutputStream(fileToAdd)) try { val newSafePointHeight = Math.max(0, height - rollbackDepth) + logger.info(s"new safe point height: ${newSafePointHeight}") + logger.info(s"write to file root node with hash: ${Algos.encode(rootNode.hash)}") val newSafePointSerialized = Ints.toByteArray(newSafePointHeight) val fileToDelete = new File(rootsPath.getAbsolutePath ++ s"/${newSafePointHeight - rollbackDepth}") if (fileToDelete.exists()) fileToDelete.delete() @@ -73,11 +78,14 @@ object RootNodesStorage { val newRootNode = insertionInfo .foldLeft(avlTree) { case (tree, (height, (toInsert, toDelete))) => + logger.info(s"Previous tree hash: ${Algos.encode(tree.rootNode.hash)}") val newTree = tree.insertAndDeleteMany( StorageVersion @@ Random.randomBytes(), toInsert, toDelete ) + logger.info(s"Current safe point: ${safePointHeight}") + logger.info(s"After insertion at height ${height} state root: ${Algos.encode(newTree.rootNode.hash)}") if (height == currentSafePoint + rollbackDepth) { batch.put(Ints.toByteArray(currentSafePoint + rollbackDepth), NodeSerilalizer.toBytes(newTree.rootNode)) } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 0c4ae8dfec..f8e3f4915e 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -3,16 +3,24 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ +import com.google.common.base.Charsets +import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews +import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq @@ -25,8 +33,12 @@ class MemoryPool(settings: EncryAppSettings, var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, networkTimeProvider) + var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + var canProcessTransactions: Boolean = false + var chainSynced: Boolean = false + override def preStart(): Unit = { logger.debug(s"Starting MemoryPool. Initializing all schedulers") context.system.scheduler.schedule( @@ -47,6 +59,47 @@ class MemoryPool(settings: EncryAppSettings, def disableTransactionsProcessor: Receive = auxiliaryReceive(MemoryPoolStateType.NotProcessingNewTransactions) def transactionsProcessor(currentNumberOfProcessedTransactions: Int): Receive = { + case DataFromPeer(message, remote) => + message match { + case RequestModifiersNetworkMessage((_, requestedIds)) => + val modifiersIds: Seq[Transaction] = requestedIds + .map(Algos.encode) + .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } + .flatten + logger.debug( + s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." + ) + context.parent ! ResponseFromLocal( + remote, + Transaction.modifierTypeId, + modifiersIds.map(tx => tx.id -> tx.bytes).toMap + ) + case InvNetworkMessage((_, txs)) => + val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) + if (notYetRequestedTransactions.nonEmpty) { + sender ! RequestFromLocal(Some(remote), Transaction.modifierTypeId, notYetRequestedTransactions.toList) + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" Not yet requested ids size is ${notYetRequestedTransactions.size}." + ) + } else + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" There are no not yet requested ids." + ) + + case InvNetworkMessage(invData) => + logger.debug( + s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions." + ) + + case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") + } + + case IsChainSynced(info) => chainSynced = info + case NewTransaction(transaction) => val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) @@ -134,6 +187,18 @@ class MemoryPool(settings: EncryAppSettings, case message => logger.debug(s"MemoryPool got unhandled message $message.") } + + def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { + case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => + bloomFilterForTransactionsIds.put(Algos.encode(id)) + id + } + + def initBloomFilter: BloomFilter[String] = BloomFilter.create( + Funnels.stringFunnel(Charsets.UTF_8), + settings.mempool.bloomFilterCapacity, + settings.mempool.bloomFilterFailureProbability + ) } object MemoryPool { diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 23a0fbbc00..c0055dd99b 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -6,13 +6,13 @@ import akka.actor.{Actor, Props} import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter} +import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -49,7 +49,6 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case DataFromPeer(message, remote) => message match { - case RequestModifiersNetworkMessage((_, requestedIds)) => val modifiersIds: Seq[Transaction] = requestedIds .map(Algos.encode) @@ -59,7 +58,11 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." ) - + context.parent ! ResponseFromLocal( + remote, + Transaction.modifierTypeId, + modifiersIds.map(tx => tx.id -> tx.bytes).toMap + ) case InvNetworkMessage((_, txs)) => val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) if (notYetRequestedTransactions.nonEmpty) { diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index 4268635170..c9c7cc571c 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -4,6 +4,7 @@ import TransactionProto.TransactionProtoMessage import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings @@ -11,7 +12,9 @@ import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.util.{Failure, Success, Try} class TransactionsValidator(settings: EncryAppSettings, @@ -21,26 +24,22 @@ class TransactionsValidator(settings: EncryAppSettings, with StrictLogging { override def receive(): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - typeId match { - case Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { - case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) - case Success(tx) => - logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") - context.parent ! BanPeer(remote.socketAddress, SyntacticallyInvalidTransaction) - context.parent ! InvalidModifier(id) - case Failure(ex) => - context.parent ! BanPeer(remote.socketAddress, CorruptedSerializedBytes) - context.parent ! InvalidModifier(id) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - } + case DataFromPeer(ModifiersNetworkMessage(data), remote) if data._1 == Transaction.modifierTypeId => + data._2.foreach { + case (id, bytes) => + Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { + case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) + case Success(tx) => + logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") + context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) + context.parent ! InvalidModifier(id) + case Failure(ex) => + context.parent ! BanPeer(remote, CorruptedSerializedBytes) + context.parent ! InvalidModifier(id) + logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } - } + } } - } object TransactionsValidator { diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 9ab46c63ef..1dec310e47 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -446,7 +446,7 @@ object AvlTree extends StrictLogging { avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V] ): AvlTree[K, V] = { - rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) + rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ -1) new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) } From ad844c62c3de2749b39f608f7d9e802af6f8dc6a Mon Sep 17 00:00:00 2001 From: aleksandr Date: Sat, 7 Mar 2020 11:41:30 +0300 Subject: [PATCH 066/119] miner pipeline --- src/main/scala/encry/Starter.scala | 6 +- src/main/scala/encry/local/miner/Miner.scala | 83 +++++++++++-------- .../scala/encry/network/DeliveryManager.scala | 4 +- .../scala/encry/nvg/IntermediaryNVH.scala | 41 +++------ .../scala/encry/nvg/nvhg/NodeViewHolder.scala | 57 ++++++------- .../view/mempool/IntermediaryMempool.scala | 30 ++++--- .../view/mempool/TransactionsValidator.scala | 9 +- 7 files changed, 114 insertions(+), 116 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 265514609c..587b5bfff3 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -411,8 +411,6 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") - val miner: ActorRef = - context.system.actorOf(Miner.props(dataHolderForApi, influxRef, newSettings), "miner") // lazy val memoryPool: ActorRef = context.system.actorOf( // MemoryPool // .props(newSettings, timeProvider, miner, influxRef) @@ -446,9 +444,11 @@ class Starter(settings: EncryAppSettings, ) val memoryPool = context.system.actorOf( - IntermediaryMempool.props(newSettings, timeProvider, miner, influxRef, networkRouter) + IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) + val miner: ActorRef = + context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index c61d078e4a..b43d9c6453 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -13,14 +13,15 @@ import encry.consensus.{CandidateBlock, EncrySupplyController, EquihashPowScheme import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.network.DeliveryManager +import encry.network.DeliveryManager.{BlockchainStatus, FullBlockChainIsSynced} +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -40,6 +41,8 @@ import scala.collection._ import scala.concurrent.duration._ class Miner(dataHolder: ActorRef, + mempool: ActorRef, + nvh: ActorRef, influx: Option[ActorRef], settings: EncryAppSettings) extends Actor with StrictLogging { @@ -62,6 +65,7 @@ class Miner(dataHolder: ActorRef, override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[ClIMiner]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) context.system.scheduler.schedule(5.seconds, 5.seconds)( influx.foreach(_ ! InfoAboutTransactionsFromMiner(transactionsPool.size)) ) @@ -79,7 +83,10 @@ class Miner(dataHolder: ActorRef, def needNewCandidate(b: Block): Boolean = !candidateOpt.flatMap(_.parentOpt).map(_.id).exists(id => Algos.encode(id) == Algos.encode(b.header.id)) - override def receive: Receive = if (settings.node.mining && syncingDone) miningEnabled else miningDisabled + override def receive: Receive = { + logger.info(s"settings.node.mining: ${settings.node.mining}. syncingDone: ${syncingDone}") + if (settings.node.mining && syncingDone) miningEnabled else miningDisabled + } def mining: Receive = { case StartMining if context.children.nonEmpty & syncingDone => @@ -113,7 +120,7 @@ class Miner(dataHolder: ActorRef, s" from worker $workerIdx with nonce: ${block.header.nonce}.") logger.debug(s"Set previousSelfMinedBlockId: ${Algos.encode(block.id)}") killAllWorkers() - //context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) + nvh ! LocallyGeneratedModifier(block) if (settings.influxDB.isDefined) { context.actorSelection("/user/statsSender") ! MiningEnd(block.header, workerIdx, context.children.size) context.actorSelection("/user/statsSender") ! MiningTime(System.currentTimeMillis() - startTime) @@ -131,9 +138,11 @@ class Miner(dataHolder: ActorRef, def miningDisabled: Receive = { case EnableMining => + logger.info("Enable mining on miner!") context.become(miningEnabled) self ! StartMining case FullBlockChainIsSynced => + logger.info("Set syncingDone to true") syncingDone = true if (settings.node.mining) self ! EnableMining case DisableMining | SemanticallySuccessfulModifier(_) => @@ -144,7 +153,7 @@ class Miner(dataHolder: ActorRef, logger.info(s"Got new block. Starting to produce candidate at height: ${mod.header.height + 1} " + s"at ${dateFormat.format(new Date(System.currentTimeMillis()))}") produceCandidate() - case SemanticallySuccessfulModifier(_) => + case SemanticallySuccessfulModifier(_) => logger.info("Got new block. But needNewCandidate - false") } def receiverCandidateBlock: Receive = { @@ -160,7 +169,9 @@ class Miner(dataHolder: ActorRef, } def chainEvents: Receive = { - case FullBlockChainIsSynced => syncingDone = true + case FullBlockChainIsSynced => + logger.info("Set syncingDone on miner to true") + syncingDone = true } def procCandidateBlock(c: CandidateBlock): Unit = { @@ -216,31 +227,33 @@ class Miner(dataHolder: ActorRef, candidate } - def produceCandidate(): Unit = - context.actorSelection("/user/nodeViewHolder") ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] { - nodeView => - val producingStartTime: Time = System.currentTimeMillis() - startTime = producingStartTime - val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) - bestHeaderOpt match { - case Some(h) => logger.info(s"Best header at height ${h.height}") - case None => logger.info(s"No best header opt") - } - val candidate: CandidateEnvelope = - if ((bestHeaderOpt.isDefined && - (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { - logger.info(s"Starting candidate generation at " + - s"${dateFormat.format(new Date(System.currentTimeMillis()))}") - if (settings.influxDB.isDefined) - context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) - logger.info("Going to calculate last block:") - val envelope: CandidateEnvelope = - CandidateEnvelope - .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) - envelope - } else CandidateEnvelope.empty - candidate + def produceCandidate(): Unit = { + val lambda = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => + { + val producingStartTime: Time = System.currentTimeMillis() + startTime = producingStartTime + val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) + bestHeaderOpt match { + case Some(h) => logger.info(s"Best header at height ${h.height}") + case None => logger.info(s"No best header opt") + } + val candidate: CandidateEnvelope = + if ((bestHeaderOpt.isDefined && + (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { + logger.info(s"Starting candidate generation at " + + s"${dateFormat.format(new Date(System.currentTimeMillis()))}") + if (settings.influxDB.isDefined) + context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) + logger.info("Going to calculate last block:") + val envelope: CandidateEnvelope = + CandidateEnvelope + .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) + envelope + } else CandidateEnvelope.empty + candidate } + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] (lambda, self) + } } object Miner { @@ -276,6 +289,10 @@ object Miner { "candidateBlock" -> r.candidateBlock.map(_.asJson).getOrElse("None".asJson) ).asJson - def props(dataHolder: ActorRef, influx: Option[ActorRef], settings: EncryAppSettings): Props = - Props(new Miner(dataHolder, influx, settings)) + def props(dataHolder: ActorRef, + mempool: ActorRef, + nvh: ActorRef, + influx: Option[ActorRef], + settings: EncryAppSettings): Props = + Props(new Miner(dataHolder, mempool, nvh, influx, settings)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index dd7f913792..64ade81e21 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -511,7 +511,9 @@ object DeliveryManager { case object CheckPayloadsToDownload - final case object FullBlockChainIsSynced + trait BlockchainStatus + + final case object FullBlockChainIsSynced extends BlockchainStatus final case class CheckModifiersWithQueueSize(size: Int) extends AnyVal diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 5947d99e8a..8480ba5681 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,48 +1,29 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{ - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} -import encry.nvg.fast.sync.SnapshotProcessor.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} import encry.nvg.nvhg.NodeViewHolder +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -107,6 +88,7 @@ class IntermediaryNVH( case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg + case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg @@ -125,6 +107,7 @@ class IntermediaryNVH( case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg @ RolledBackTransactions(_) => //+ to memory pool case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 6dcf1745e5..d9121897d4 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -2,61 +2,48 @@ package encry.nvg.nvhg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} +import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.fast.sync.SnapshotProcessor.{ - FastSyncDone, - FastSyncFinished, - HeaderChainIsSynced, - RemoveRedundantManifestIds, - SnapshotChunk, - TreeChunks -} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, FastSyncFinished, HeaderChainIsSynced, RemoveRedundantManifestIds, SnapshotChunk, TreeChunks} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.nvhg.NodeViewHolder.{ - NodeView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader, - UpdateInformation -} +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, NodeView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError +import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} +import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -121,6 +108,13 @@ class NodeViewHolder( s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) + case GetDataFromCurrentView(f, sender) => + logger.info("Receive GetDataFromCurrentView on nvh") + f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { + case resultFuture: Future[_] => resultFuture.pipeTo(sender) + case result => sender ! result + } + case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => val startTime: Long = System.currentTimeMillis() logger.info(s"Got a signal about finishing fast sync process.") @@ -290,7 +284,7 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.parent ! SemanticallySuccessfulModifier(modToApply) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) @@ -359,16 +353,17 @@ class NodeViewHolder( logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) if (newHistory.isFullChainSynced) { - logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") + logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() context.parent ! FullBlockChainIsSynced + context.system.eventStream.publish(FullBlockChainIsSynced) } updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.parent ! SemanticallySuccessfulModifier(modifier) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e: Throwable) => @@ -525,6 +520,8 @@ object NodeViewHolder { case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome + case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A, sender: ActorRef) + final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId] diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 62b41dc8bb..acf38c9982 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -11,12 +11,11 @@ import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, influxReference: Option[ActorRef], networkRouter: ActorRef) extends Actor @@ -35,27 +34,26 @@ class IntermediaryMempool(settings: EncryAppSettings, } override def receive(): Receive = { - case msg @ InvalidModifier(_) => // to nvsh - case msg @ BanPeer(_, _) => // to peersKeeper - case msg @ ValidatedModifierFromNetwork(_) => // to influx - case msg @ TransactionsForMiner(_) => minerReference ! msg// to miner - case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool - case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool - case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor - case msg @ RequestFromLocal(_, _, _) => // to network - case msg @ ModifierFromNetwork(_, _, _, _) => txValidator ! msg - case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor - case msg @ IsChainSynced(_) => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: ValidatedModifierFromNetwork => // to influx + case msg: RolledBackTransactions => memoryPool ! msg // to mempool + case msg: ModifiersForValidating => memoryPool ! msg // to mempool + case msg: DataFromPeer => mempoolProcessor ! msg // to mempool processor + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg // to mempool processor + case msg: IsChainSynced => mempoolProcessor ! msg } } object IntermediaryMempool { + def props(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, influxReference: Option[ActorRef], - networkRouter: ActorRef) = - Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference, networkRouter)) + networkRouter: ActorRef): Props = + Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index c9c7cc571c..fe00f24a46 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -7,10 +7,11 @@ import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, Syntacticall import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -32,10 +33,10 @@ class TransactionsValidator(settings: EncryAppSettings, case Success(tx) => logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) - context.parent ! InvalidModifier(id) + context.parent ! InvalidTransaction(id) case Failure(ex) => context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifier(id) + context.parent ! InvalidModifierBytes(id) logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } } @@ -48,7 +49,7 @@ object TransactionsValidator { typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) - final case class InvalidModifier(ids: ModifierId) extends AnyVal + final case class InvalidTransaction(ids: ModifierId) extends AnyVal def props(settings: EncryAppSettings, memPool: ActorRef, ntp: NetworkTimeProvider): Props = Props(new TransactionsValidator(settings, memPool, ntp)) From 901439c10f3cbe05b2f7679167afc52d67b23605 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 11:19:18 +0300 Subject: [PATCH 067/119] add logging + fix re requesting --- src/main/scala/encry/network/DM.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 72804c6a5c..d8cf9f126f 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -40,18 +40,21 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifier += toKey(id) } else logger.info("Receive spam!") } - case RequestSent(peer, modTypeId, modId) => + case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) + case RequestSent(_, _, _) => //do nothing case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + logger.info(s"Re-request modifier ${Algos.encode(modId)}") context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! AwaitingRequest(peer, modTypeId, modId, attempts + 1) ) - case AwaitingRequest(peer, _, modId, _) => - logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") + case AwaitingRequest(peer, _, modId, attempts) => + logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) From b61733f114adf2c25a694afc6ba359dc00ecfb6e Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 11:27:55 +0300 Subject: [PATCH 068/119] remove mod from expected after failed re-requesting --- src/main/scala/encry/network/DM.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index d8cf9f126f..4a4e0510ce 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -55,6 +55,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case AwaitingRequest(peer, _, modId, attempts) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") + expectedModifiers -= toKey(modId) case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) From 55a5414000082852a767cfadda6ebd432864ae00 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 11:53:30 +0300 Subject: [PATCH 069/119] state/history separation frame --- src/main/scala/encry/nvg/ModifiersCache.scala | 17 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 6 +- .../nvg/nvhg/NodeViewHolderController.scala | 179 +++++++++++++++++- .../scala/encry/nvg/nvhg/StateApplier.scala | 6 +- 4 files changed, 189 insertions(+), 19 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 07ed709065..24006111cb 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -62,7 +62,7 @@ object ModifiersCache extends StrictLogging { } def popCandidate(history: History): List[PersistentModifier] = synchronized { - findCandidateKey(history).flatMap(k => remove(k)) + findCandidateKey(history).take(1).flatMap(k => remove(k)) } override def toString: String = cache.keys.map(key => Algos.encode(key.toArray)).mkString(",") @@ -91,13 +91,6 @@ object ModifiersCache extends StrictLogging { List.empty[Key] } - def findApplicablePayloadAtHeight(height: Int): List[Key] = { - history.headerIdsAtHeight(height).view.flatMap(history.getHeaderById).collect { - case header: Header if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => - new mutable.WrappedArray.ofByte(header.payloadId) - } - }.toList - def exhaustiveSearch: List[Key] = List(cache.find { case (k, v) => @@ -109,14 +102,6 @@ object ModifiersCache extends StrictLogging { } }).collect { case Some(v) => v._1 } - @tailrec - def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, - prevKeys: List[Key] = List.empty[Key]): List[Key] = { - val payloads = findApplicablePayloadAtHeight(atHeight) - if (payloads.nonEmpty) applicableBestPayloadChain(atHeight + 1, prevKeys ++ payloads) - else prevKeys - } - val bestHeadersIds: List[Key] = { headersCollection.get(history.getBestHeaderHeight + 1) match { case Some(value) => diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala index 3f47124fd2..4e75ae2909 100644 --- a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala +++ b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala @@ -1,5 +1,9 @@ package encry.nvg.nvhg -class HistoryApplier { +import akka.actor.Actor +class HistoryApplier extends Actor { + override def receive: Receive = { + case _ => + } } diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala index f64e04a22f..80b43b6539 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala @@ -1,5 +1,182 @@ package encry.nvg.nvhg -class NodeViewHolderController { +import java.io.File +import akka.actor.{ Actor, ActorRef } +import cats.syntax.option._ +import com.typesafe.scalalogging.StrictLogging +import encry.nvg.ModifiersCache +import encry.nvg.ModifiersValidator.ValidatedModifier +import encry.nvg.nvhg.NodeViewHolder.NodeView +import encry.nvg.nvhg.NodeViewHolderController.{ ApplicableModifier, GetNewApplicable, HistoryIsReady, StateIsReady } +import encry.settings.EncryAppSettings +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.NetworkTimeProvider +import encry.view.history.History +import encry.view.state.UtxoState +import encry.view.wallet.EncryWallet +import org.apache.commons.io.FileUtils +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ADDigest + +import scala.collection.mutable + +class NodeViewHolderController( + settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging { + + var nodeView: NodeView = restoreState().getOrElse(genesisState) + + val historyApplier: ActorRef = Actor.noSender + + val stateApplier: ActorRef = Actor.noSender + + override def receive: Receive = mainReceive.orElse(processModifierFromNetwork) + + def mainReceive: Receive = { + case GetNewApplicable => + val mod: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) + mod.headOption.foreach { mod: PersistentModifier => + historyApplier ! ApplicableModifier(mod) + stateApplier ! ApplicableModifier(mod) + logger.info(s"Applicable modifier ${mod.encodedId} of type ${mod.modifierTypeId} sent to history and state.") + context.become( + awaitApplication(historyIsReady = false, stateIsReady = false).orElse(processModifierFromNetwork) + ) + } + } + + def awaitApplication(historyIsReady: Boolean, stateIsReady: Boolean): Receive = { + case HistoryIsReady => + if (stateIsReady) { + self ! GetNewApplicable + context.become(mainReceive.orElse(processModifierFromNetwork)) + } else context.become(awaitApplication(historyIsReady = true, stateIsReady).orElse(processModifierFromNetwork)) + case StateIsReady => + if (historyIsReady) { + self ! GetNewApplicable + context.become(mainReceive.orElse(processModifierFromNetwork)) + } else context.become(awaitApplication(historyIsReady, stateIsReady = true).orElse(processModifierFromNetwork)) + } + + def processModifierFromNetwork: Receive = { + case ValidatedModifier(modifier: PersistentModifier) => + val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(wrappedKey) + if (isInHistory || isInCache) + logger.info( + s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + + s"contains in cache: $isInCache, contains in history: $isInHistory." + ) + else { + ModifiersCache.put(wrappedKey, modifier, nodeView.history) + self ! GetNewApplicable + } + } + + def genesisState: NodeView = { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdir() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") + val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) + val history: History = History.readOrGenerate(settings, ntp) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + NodeView(history, state, wallet) + } + + def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = + if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) + try { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val history: History = History.readOrGenerate(settings, ntp) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + history, + influxRef + ) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + NodeView(history, state, wallet).some + } catch { + case ex: Throwable => + logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") + new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) + genesisState.some + } else { + none + } + + def getRecreatedState( + version: Option[VersionTag] = none, + digest: Option[ADDigest] = none, + influxRef: Option[ActorRef] + ): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + UtxoState.create(stateDir, rootsDir, settings, influxRef) + } + + def restoreConsistentState( + stateIn: UtxoState, + history: History, + influxRefActor: Option[ActorRef] + ): UtxoState = + (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { + case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => + logger.info(s"State and history are both empty on startup") + stateIn + case (_, None, _, _) => + logger.info( + s"State and history are inconsistent." + + s" History is empty on startup, rollback state to genesis." + ) + getRecreatedState(influxRef = influxRefActor) + case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => + val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) + val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) + logger.info( + s"State and history are inconsistent." + + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + + s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" + ) + val additionalBlocks = + (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") + rollbackId + .map(_ => state.restore(additionalBlocks).get) + .getOrElse(getRecreatedState(influxRef = influxRefActor)) + } +} + +object NodeViewHolderController { + + final case class ApplicableModifier(modifier: PersistentModifier) + case object HistoryIsReady + case object StateIsReady + case object GetNewApplicable } diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala index 5316a2b46a..a94bfee293 100644 --- a/src/main/scala/encry/nvg/nvhg/StateApplier.scala +++ b/src/main/scala/encry/nvg/nvhg/StateApplier.scala @@ -1,5 +1,9 @@ package encry.nvg.nvhg -class StateApplier { +import akka.actor.Actor +class StateApplier extends Actor { + override def receive: Receive = { + case _ => + } } From aafceb3e1779fd85177bcd1992da8b7fd91984ec Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 11:55:08 +0300 Subject: [PATCH 070/119] state/history separation frame --- src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index d9121897d4..91afd77947 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -39,7 +39,6 @@ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} - import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.Future import scala.concurrent.duration._ From da7a7c718b7f1f2e584508b3cf458940194aa1e9 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 13:34:43 +0300 Subject: [PATCH 071/119] removed history\state separation --- .../encry/api/http/DataHolderForApi.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 4 +- src/main/scala/encry/network/DM.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 42 ++-- src/main/scala/encry/nvg/ModifiersCache.scala | 2 - .../scala/encry/nvg/ModifiersValidator.scala | 4 +- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +- .../encry/nvg/{nvhg => }/NodeViewHolder.scala | 31 ++- .../nvg/fast/sync/SnapshotDownloader.scala | 2 +- .../nvg/fast/sync/SnapshotProcessor.scala | 2 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 9 - .../nvg/nvhg/NodeViewHolderController.scala | 182 ------------------ .../scala/encry/nvg/nvhg/StateApplier.scala | 9 - .../scala/encry/view/mempool/MemoryPool.scala | 2 +- 16 files changed, 67 insertions(+), 248 deletions(-) rename src/main/scala/encry/nvg/{nvhg => }/NodeViewHolder.scala (95%) delete mode 100644 src/main/scala/encry/nvg/nvhg/HistoryApplier.scala delete mode 100644 src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala delete mode 100644 src/main/scala/encry/nvg/nvhg/StateApplier.scala diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index f29e856311..935b592d8d 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,7 +21,7 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.nvhg.NodeViewHolder.NodeViewChange +import encry.nvg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b43d9c6453..8512ddfd23 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -15,8 +15,8 @@ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager import encry.network.DeliveryManager.{BlockchainStatus, FullBlockChainIsSynced} -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 4a4e0510ce..b766d63723 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -14,7 +14,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ -import encry.nvg.nvhg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable import scala.concurrent.Future diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a98a3393f5..9ceb8a9576 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -18,7 +18,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 9313e51524..e3d4b4bbbb 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -33,7 +33,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.nvhg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import scala.util.Try diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8480ba5681..3539027feb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,29 +1,49 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} -import encry.nvg.nvhg.NodeViewHolder -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder.{ + GetDataFromCurrentView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, + RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 24006111cb..9a2df13159 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -7,8 +7,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.annotation.tailrec import scala.collection.concurrent.TrieMap import scala.collection.immutable.SortedMap import scala.collection.mutable diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 5dfa628e8b..e3990ec576 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } -import encry.nvg.nvhg.NodeViewHolder.SyntacticallyFailedModification +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -55,8 +55,6 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } } - - private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 75d88241b7..8415387d83 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,22 +1,26 @@ package encry.nvg -import akka.actor.{Actor, Cancellable, Props} +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.nvhg.NodeViewHolder -import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, UpdateHistoryReader} +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala similarity index 95% rename from src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala rename to src/main/scala/encry/nvg/NodeViewHolder.scala index 91afd77947..237cf9959c 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,24 +1,23 @@ -package encry.nvg.nvhg +package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, FastSyncFinished, HeaderChainIsSynced, RemoveRedundantManifestIds, SnapshotChunk, TreeChunks} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, NodeView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader, UpdateInformation} +import encry.nvg.fast.sync.SnapshotProcessor._ import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -27,22 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{IndexedSeq, Seq, mutable} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } + +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -63,7 +63,7 @@ class NodeViewHolder( context.system.scheduler.schedule(1.seconds, 10.seconds) { logger.info( - s"\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"\n\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + @@ -111,7 +111,7 @@ class NodeViewHolder( logger.info("Receive GetDataFromCurrentView on nvh") f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender) - case result => sender ! result + case result => sender ! result } case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => @@ -220,7 +220,6 @@ class NodeViewHolder( val additionalBlocks: List[Block] = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { case (blocks: List[Block], height: Int) => - //todo get best header id instead of best header val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight diff --git a/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala index bcc4ed26c5..ceba485782 100644 --- a/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, Cancellable} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest import encry.nvg.fast.sync.SnapshotProcessor.{BroadcastManifestRequestMessage, RequiredManifestHeightAndId} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.view.fast.sync.{SnapshotDownloadController, SnapshotHolder} import encry.view.history.HistoryReader diff --git a/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala index 945c6cb544..5c6fd18db7 100644 --- a/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala @@ -11,7 +11,7 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory import encry.nvg.fast.sync.SnapshotProcessor.{DropProcessedCount, HeaderChainIsSynced, RemoveRedundantManifestIds, TreeChunks} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{StorageKey, StorageValue} import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala deleted file mode 100644 index 4e75ae2909..0000000000 --- a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala +++ /dev/null @@ -1,9 +0,0 @@ -package encry.nvg.nvhg - -import akka.actor.Actor - -class HistoryApplier extends Actor { - override def receive: Receive = { - case _ => - } -} diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala deleted file mode 100644 index 80b43b6539..0000000000 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala +++ /dev/null @@ -1,182 +0,0 @@ -package encry.nvg.nvhg - -import java.io.File - -import akka.actor.{ Actor, ActorRef } -import cats.syntax.option._ -import com.typesafe.scalalogging.StrictLogging -import encry.nvg.ModifiersCache -import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.nvhg.NodeViewHolder.NodeView -import encry.nvg.nvhg.NodeViewHolderController.{ ApplicableModifier, GetNewApplicable, HistoryIsReady, StateIsReady } -import encry.settings.EncryAppSettings -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.NetworkTimeProvider -import encry.view.history.History -import encry.view.state.UtxoState -import encry.view.wallet.EncryWallet -import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ADDigest - -import scala.collection.mutable - -class NodeViewHolderController( - settings: EncryAppSettings, - ntp: NetworkTimeProvider, - influxRef: Option[ActorRef] -) extends Actor - with StrictLogging { - - var nodeView: NodeView = restoreState().getOrElse(genesisState) - - val historyApplier: ActorRef = Actor.noSender - - val stateApplier: ActorRef = Actor.noSender - - override def receive: Receive = mainReceive.orElse(processModifierFromNetwork) - - def mainReceive: Receive = { - case GetNewApplicable => - val mod: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) - mod.headOption.foreach { mod: PersistentModifier => - historyApplier ! ApplicableModifier(mod) - stateApplier ! ApplicableModifier(mod) - logger.info(s"Applicable modifier ${mod.encodedId} of type ${mod.modifierTypeId} sent to history and state.") - context.become( - awaitApplication(historyIsReady = false, stateIsReady = false).orElse(processModifierFromNetwork) - ) - } - } - - def awaitApplication(historyIsReady: Boolean, stateIsReady: Boolean): Receive = { - case HistoryIsReady => - if (stateIsReady) { - self ! GetNewApplicable - context.become(mainReceive.orElse(processModifierFromNetwork)) - } else context.become(awaitApplication(historyIsReady = true, stateIsReady).orElse(processModifierFromNetwork)) - case StateIsReady => - if (historyIsReady) { - self ! GetNewApplicable - context.become(mainReceive.orElse(processModifierFromNetwork)) - } else context.become(awaitApplication(historyIsReady, stateIsReady = true).orElse(processModifierFromNetwork)) - } - - def processModifierFromNetwork: Receive = { - case ValidatedModifier(modifier: PersistentModifier) => - val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) - val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) - val isInCache: Boolean = ModifiersCache.contains(wrappedKey) - if (isInHistory || isInCache) - logger.info( - s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + - s"contains in cache: $isInCache, contains in history: $isInHistory." - ) - else { - ModifiersCache.put(wrappedKey, modifier, nodeView.history) - self ! GetNewApplicable - } - } - - def genesisState: NodeView = { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdir() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - NodeView(history, state, wallet) - } - - def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = - if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) - try { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, settings, influxRef), - history, - influxRef - ) - history.updateIdsForSyncInfo() - logger.info(s"History best block height: ${history.getBestBlockHeight}") - logger.info(s"History best header height: ${history.getBestHeaderHeight}") - NodeView(history, state, wallet).some - } catch { - case ex: Throwable => - logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") - new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - genesisState.some - } else { - none - } - - def getRecreatedState( - version: Option[VersionTag] = none, - digest: Option[ADDigest] = none, - influxRef: Option[ActorRef] - ): UtxoState = { - val dir: File = UtxoState.getStateDir(settings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - UtxoState.create(stateDir, rootsDir, settings, influxRef) - } - - def restoreConsistentState( - stateIn: UtxoState, - history: History, - influxRefActor: Option[ActorRef] - ): UtxoState = - (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { - case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => - logger.info(s"State and history are both empty on startup") - stateIn - case (_, None, _, _) => - logger.info( - s"State and history are inconsistent." + - s" History is empty on startup, rollback state to genesis." - ) - getRecreatedState(influxRef = influxRefActor) - case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => - val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) - val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) - logger.info( - s"State and history are inconsistent." + - s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" - ) - val additionalBlocks = - (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") - rollbackId - .map(_ => state.restore(additionalBlocks).get) - .getOrElse(getRecreatedState(influxRef = influxRefActor)) - } -} - -object NodeViewHolderController { - - final case class ApplicableModifier(modifier: PersistentModifier) - case object HistoryIsReady - case object StateIsReady - case object GetNewApplicable -} diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala deleted file mode 100644 index a94bfee293..0000000000 --- a/src/main/scala/encry/nvg/nvhg/StateApplier.scala +++ /dev/null @@ -1,9 +0,0 @@ -package encry.nvg.nvhg - -import akka.actor.Actor - -class StateApplier extends Actor { - override def receive: Receive = { - case _ => - } -} diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index f8e3f4915e..a9c751ff7b 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -9,7 +9,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews From 140c4bf44a387e3dbafbddc2aca2e99bc0fe72f7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 13:53:34 +0300 Subject: [PATCH 072/119] added messages to mempool from nvh --- src/main/scala/encry/Starter.scala | 59 ++++++------------- .../scala/encry/nvg/IntermediaryNVH.scala | 12 ++-- 2 files changed, 25 insertions(+), 46 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 587b5bfff3..0128ea4043 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,7 +1,6 @@ package encry import java.net.InetSocketAddress - import akka.actor.{ Actor, ActorRef } import akka.http.scaladsl.Http import cats.Functor @@ -24,12 +23,8 @@ import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender import encry.utils.{ Mnemonic, NetworkTimeProvider } -import encry.view.mempool.MemoryPool -import encry.utils.{Mnemonic, NetworkTimeProvider} -import encry.view.NodeViewHolder -import encry.view.mempool.{IntermediaryMempool, MemoryPool} +import encry.view.mempool.IntermediaryMempool import encry.view.wallet.AccountManager - import scala.concurrent.Future import scala.io.StdIn import scala.util.{ Failure, Success, Try } @@ -411,27 +406,9 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") -// lazy val memoryPool: ActorRef = context.system.actorOf( -// MemoryPool -// .props(newSettings, timeProvider, miner, influxRef) -// .withDispatcher("mempool-dispatcher") -// ) -// val nodeViewHolder: ActorRef = context.system.actorOf( -// NodeViewHolder -// .props(memoryPool, influxRef, dataHolderForApi, newSettings) -// .withDispatcher("nvh-dispatcher"), -// "nodeViewHolder" -// ) if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) -// context.system.actorOf( -// NodeViewSynchronizer -// .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) -// .withDispatcher("nvsh-dispatcher"), -// "nodeViewSynchronizer" -// ) - val networkRouter = context.system.actorOf( NetworkRouter .props(networkSettings, settings.blackList) @@ -439,14 +416,12 @@ class Starter(settings: EncryAppSettings, "networkRouter" ) - val nvhRouter = context.system.actorOf( - IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) - ) - - val memoryPool = context.system.actorOf( + val memoryPool: ActorRef = context.system.actorOf( IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) - + val nvhRouter: ActorRef = context.system.actorOf( + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool) + ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") if (newSettings.node.mining) miner ! StartMining @@ -461,15 +436,17 @@ class Starter(settings: EncryAppSettings, } object Starter { - final case class InitNodeResult(mnemonic: String, - walletPassword: String, - offlineGeneration: Boolean, - fastSync: Boolean, - snapshotCreation: Boolean, - peers: List[InetSocketAddress], - connectWithOnlyKnownPeers: Boolean, - nodePass: String = "", - nodeName: String, - declaredAddr: Option[InetSocketAddress], - bindAddr: InetSocketAddress) + final case class InitNodeResult( + mnemonic: String, + walletPassword: String, + offlineGeneration: Boolean, + fastSync: Boolean, + snapshotCreation: Boolean, + peers: List[InetSocketAddress], + connectWithOnlyKnownPeers: Boolean, + nodePass: String = "", + nodeName: String, + declaredAddr: Option[InetSocketAddress], + bindAddr: InetSocketAddress + ) } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 3539027feb..07b78aa7d3 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -50,7 +50,8 @@ class IntermediaryNVH( settings: EncryAppSettings, intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, - influxRef: Option[ActorRef] + influxRef: Option[ActorRef], + mempoolRef: ActorRef ) extends Actor with StrictLogging { @@ -121,11 +122,11 @@ class IntermediaryNVH( case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => //+ to miner + case msg @ FullBlockChainIsSynced => mempoolRef ! msg + case msg @ RolledBackTransactions(_) => mempoolRef ! msg case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg case msg @ RollbackSucceed(_) => @@ -142,6 +143,7 @@ object IntermediaryNVH { settings: EncryAppSettings, intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, - influxRef: Option[ActorRef] - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef)) + influxRef: Option[ActorRef], + mempoolRef: ActorRef + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef)) } From 801e27b05a91e1e799248b5518334ba0e299c463 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 14:20:17 +0300 Subject: [PATCH 073/119] added forwarding instead of ref in message --- src/main/scala/encry/local/miner/Miner.scala | 2 +- src/main/scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 4 ++-- src/main/scala/encry/view/mempool/MemoryPool.scala | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 8512ddfd23..b2b45cd2f4 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -252,7 +252,7 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] (lambda, self) + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda) } } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 07b78aa7d3..d789ef0ed3 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -128,7 +128,7 @@ class IntermediaryNVH( case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg + case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 237cf9959c..f628a3f571 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -107,7 +107,7 @@ class NodeViewHolder( s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) - case GetDataFromCurrentView(f, sender) => + case GetDataFromCurrentView(f) => logger.info("Receive GetDataFromCurrentView on nvh") f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender) @@ -518,7 +518,7 @@ object NodeViewHolder { case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome - case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A, sender: ActorRef) + case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A) final case class DownloadRequest( modifierTypeId: ModifierTypeId, diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index a9c751ff7b..0da47920ae 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -12,7 +12,6 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.ReceivableMessages.CompareViews import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ From a457f6684190eb72a8807f16af95a18e3b5f1474 Mon Sep 17 00:00:00 2001 From: Lior Date: Tue, 10 Mar 2020 14:24:22 +0300 Subject: [PATCH 074/119] code cleanup --- .../encry/view/mempool/IntermediaryMempool.scala | 16 +++++++--------- .../encry/view/mempool/MemoryPoolProcessor.scala | 5 ++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index acf38c9982..1181ffd9f5 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -2,6 +2,7 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} @@ -9,7 +10,6 @@ import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction @@ -36,14 +36,14 @@ class IntermediaryMempool(settings: EncryAppSettings, override def receive(): Receive = { case msg: InvalidTransaction => networkRouter ! msg case msg: BanPeer => networkRouter ! msg - case msg: ValidatedModifierFromNetwork => // to influx - case msg: RolledBackTransactions => memoryPool ! msg // to mempool - case msg: ModifiersForValidating => memoryPool ! msg // to mempool - case msg: DataFromPeer => mempoolProcessor ! msg // to mempool processor + case msg: ValidatedModifierFromNetwork => influxReference.foreach(_ ! msg) + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg case msg: RequestFromLocal => networkRouter ! msg case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg // to mempool processor - case msg: IsChainSynced => mempoolProcessor ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } @@ -56,6 +56,4 @@ object IntermediaryMempool { Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) - - final case class IsChainSynced(info: Boolean) } diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index c0055dd99b..fe9ea7dc5e 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,11 +1,10 @@ package encry.view.mempool -import java.net.InetSocketAddress - import akka.actor.{Actor, Props} import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings @@ -42,7 +41,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case TransactionProcessing(info) => canProcessTransactions = info - case IsChainSynced(info) => chainSynced = info + case FullBlockChainIsSynced(info) => chainSynced = info case CleanupBloomFilter => bloomFilterForTransactionsIds = initBloomFilter From 5033ed18d943ecc11aa4ce93d010bc425bddc53c Mon Sep 17 00:00:00 2001 From: Lior Date: Tue, 10 Mar 2020 14:36:22 +0300 Subject: [PATCH 075/119] code cleanup --- src/main/scala/encry/view/mempool/IntermediaryMempool.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 1181ffd9f5..e49ef2be36 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -8,7 +8,6 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings -import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} @@ -36,7 +35,6 @@ class IntermediaryMempool(settings: EncryAppSettings, override def receive(): Receive = { case msg: InvalidTransaction => networkRouter ! msg case msg: BanPeer => networkRouter ! msg - case msg: ValidatedModifierFromNetwork => influxReference.foreach(_ ! msg) case msg: RolledBackTransactions => memoryPool ! msg case msg: ModifiersForValidating => memoryPool ! msg case msg: DataFromPeer => mempoolProcessor ! msg @@ -54,6 +52,4 @@ object IntermediaryMempool { influxReference: Option[ActorRef], networkRouter: ActorRef): Props = Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) - - final case class TransactionsForValidating(tx: Transaction) } From 17b85ec5038b6e060b1acea9d5554c779573e3e7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 15:43:13 +0300 Subject: [PATCH 076/119] added mempool reader --- .../view/mempool/IntermediaryMempool.scala | 55 ++++---- .../scala/encry/view/mempool/MemoryPool.scala | 122 ++++++++++-------- .../view/mempool/MemoryPoolProcessor.scala | 40 +++--- .../encry/view/mempool/MemoryPoolReader.scala | 13 ++ .../view/mempool/MemoryPoolStorage.scala | 18 +-- .../view/mempool/TransactionsValidator.scala | 30 ++--- 6 files changed, 156 insertions(+), 122 deletions(-) create mode 100644 src/main/scala/encry/view/mempool/MemoryPoolReader.scala diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index e49ef2be36..3e422ae5b8 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -1,17 +1,16 @@ package encry.view.mempool -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForTxHandling } import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import encry.view.mempool.MemoryPool.{ RolledBackTransactions, TransactionProcessing } +import encry.view.mempool.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, @@ -20,36 +19,40 @@ class IntermediaryMempool(settings: EncryAppSettings, extends Actor with StrictLogging { - val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, self, influxReference), name = "mempool") - val txValidator: ActorRef = - context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), - name = "Transaction-validator") val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") - override def preStart(): Unit = { - networkRouter ! RegisterForTxHandling - } + val memoryPool: ActorRef = + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference), name = "mempool") + + val txValidator: ActorRef = + context.actorOf( + TransactionsValidator.props(settings, memoryPool, networkTimeProvider), + name = "Transaction-validator" + ) + + override def preStart(): Unit = networkRouter ! RegisterForTxHandling override def receive(): Receive = { - case msg: InvalidTransaction => networkRouter ! msg - case msg: BanPeer => networkRouter ! msg - case msg: RolledBackTransactions => memoryPool ! msg - case msg: ModifiersForValidating => memoryPool ! msg - case msg: DataFromPeer => mempoolProcessor ! msg - case msg: RequestFromLocal => networkRouter ! msg - case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg - case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } object IntermediaryMempool { - def props(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - influxReference: Option[ActorRef], - networkRouter: ActorRef): Props = + def props( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + networkRouter: ActorRef + ): Props = Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 0da47920ae..47e86b213f 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -1,32 +1,34 @@ package encry.view.mempool -import akka.actor.{Actor, ActorRef, ActorSystem, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.actor.{ Actor, ActorRef, ActorSystem, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import cats.syntax.either._ import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} +import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq -class MemoryPool(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - intermediaryMempool: ActorRef, - influxReference: Option[ActorRef]) extends Actor with StrictLogging { +class MemoryPool( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + mempoolProcessor: ActorRef +) extends Actor + with StrictLogging { import context.dispatcher @@ -42,10 +44,16 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"Starting MemoryPool. Initializing all schedulers") context.system.scheduler.schedule( settings.mempool.cleanupInterval, - settings.mempool.cleanupInterval, self, RemoveExpiredFromPool) + settings.mempool.cleanupInterval, + self, + RemoveExpiredFromPool + ) context.system.scheduler.schedule( settings.mempool.txSendingInterval, - settings.mempool.txSendingInterval, self, SendTransactionsToMiner) + settings.mempool.txSendingInterval, + self, + SendTransactionsToMiner + ) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } @@ -103,56 +111,61 @@ class MemoryPool(settings: EncryAppSettings, val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) memoryPool = newMemoryPool + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) validatedTransaction.foreach(tx => context.system.eventStream.publish(SuccessfulTransaction(tx))) logger.debug(s"MemoryPool got new transactions from remote. New pool size is ${memoryPool.size}.") if (currentNumberOfProcessedTransactions > settings.mempool.transactionsLimit) { - logger.debug(s"MemoryPool has its limit of processed transactions. " + - s"Transit to 'disableTransactionsProcessor' state." + - s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") + logger.debug( + s"MemoryPool has its limit of processed transactions. " + + s"Transit to 'disableTransactionsProcessor' state." + + s"Current number of processed transactions is $currentNumberOfProcessedTransactions." + ) canProcessTransactions = false context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + 1 - logger.debug(s"Current number of processed transactions is OK. Continue to process them..." + - s" Current number is $currentTransactionsNumber.") + logger.debug( + s"Current number of processed transactions is OK. Continue to process them..." + + s" Current number is $currentTransactionsNumber." + ) context.become(continueProcessing(currentTransactionsNumber)) } -// case CompareViews(peer, _, transactions) => -// val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) -// if (notYetRequestedTransactions.nonEmpty) { -// sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) -// logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + -// s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") -// } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + -// s" There are no not yet requested ids.") - case RolledBackTransactions(transactions) => val (newMemoryPool: MemoryPoolStorage, validatedTransactions: Seq[Transaction]) = memoryPool.validateTransactions(transactions) memoryPool = newMemoryPool - logger.debug(s"MemoryPool got rolled back transactions. New pool size is ${memoryPool.size}." + - s"Number of rolled back transactions is ${validatedTransactions.size}.") + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) + logger.debug( + s"MemoryPool got rolled back transactions. New pool size is ${memoryPool.size}." + + s"Number of rolled back transactions is ${validatedTransactions.size}." + ) if (currentNumberOfProcessedTransactions > settings.mempool.transactionsLimit) { - logger.debug(s"MemoryPool has its limit of processed transactions. " + - s"Transit to 'disableTransactionsProcessor' state." + - s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") + logger.debug( + s"MemoryPool has its limit of processed transactions. " + + s"Transit to 'disableTransactionsProcessor' state." + + s"Current number of processed transactions is $currentNumberOfProcessedTransactions." + ) canProcessTransactions = false context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + validatedTransactions.size - logger.debug(s"Current number of processed transactions is OK. Continue to process them..." + - s" Current number is $currentTransactionsNumber.") + logger.debug( + s"Current number of processed transactions is OK. Continue to process them..." + + s" Current number is $currentTransactionsNumber." + ) context.become(continueProcessing(currentTransactionsNumber)) } } def auxiliaryReceive(state: MemoryPoolStateType): Receive = { case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Block.modifierTypeId => - logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + - s"Transit to a transactionsProcessor state.") + logger.debug( + s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + + s"Transit to a transactionsProcessor state." + ) if (state == NotProcessingNewTransactions) Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) canProcessTransactions = true @@ -160,16 +173,21 @@ class MemoryPool(settings: EncryAppSettings, context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) case SemanticallySuccessfulModifier(_) => - logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + - s"while $state. Do nothing in this case.") + logger.debug( + s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + + s"while $state. Do nothing in this case." + ) case SendTransactionsToMiner => val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = memoryPool.getTransactionsForMiner memoryPool = newMemoryPool - intermediaryMempool ! TransactionsForMiner(transactionsForMiner) - logger.debug(s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + - s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}") + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) + context.parent ! TransactionsForMiner(transactionsForMiner) + logger.debug( + s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}" + ) case RemoveExpiredFromPool => memoryPool = memoryPool.filter(memoryPool.isExpired) @@ -220,6 +238,8 @@ object MemoryPool { sealed trait MemoryPoolStateType + final case class UpdateMempoolReader(reader: MemoryPoolReader) + object MemoryPoolStateType { case object ProcessingNewTransaction extends MemoryPoolStateType @@ -228,18 +248,18 @@ object MemoryPool { } - def props(settings: EncryAppSettings, - ntp: NetworkTimeProvider, - intermediaryMempool: ActorRef, - influx: Option[ActorRef]): Props = - Props(new MemoryPool(settings, ntp, intermediaryMempool, influx)) + def props( + settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influx: Option[ActorRef], + mempoolProcessor: ActorRef + ): Props = Props(new MemoryPool(settings, ntp, influx, mempoolProcessor)) class MemoryPoolPriorityQueue(settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator { + extends UnboundedStablePriorityMailbox(PriorityGenerator { case RemoveExpiredFromPool | SendTransactionsToMiner => 0 - case NewTransaction(_) => 1 - case otherwise => 2 + case NewTransaction(_) => 1 + case otherwise => 2 }) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index fe9ea7dc5e..b86b30fa20 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,19 +1,18 @@ package encry.view.mempool -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} +import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.scalalogging.StrictLogging import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced -import encry.view.mempool.MemoryPool.TransactionProcessing +import encry.view.mempool.MemoryPool.{ TransactionProcessing, UpdateMempoolReader } import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -29,30 +28,27 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) var chainSynced: Boolean = false - var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, ntp) + var memoryPoolReader: MemoryPoolReader = MemoryPoolReader.empty override def preStart(): Unit = - context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, - settings.mempool.bloomFilterCleanupInterval, - self, - CleanupBloomFilter) + context.system.scheduler.schedule( + settings.mempool.bloomFilterCleanupInterval, + settings.mempool.bloomFilterCleanupInterval, + self, + CleanupBloomFilter + ) override def receive: Receive = { - + case UpdateMempoolReader(reader) => memoryPoolReader = reader case TransactionProcessing(info) => canProcessTransactions = info - - case FullBlockChainIsSynced(info) => chainSynced = info - - case CleanupBloomFilter => - bloomFilterForTransactionsIds = initBloomFilter - + case FullBlockChainIsSynced => chainSynced = true + case CleanupBloomFilter => bloomFilterForTransactionsIds = initBloomFilter case DataFromPeer(message, remote) => message match { case RequestModifiersNetworkMessage((_, requestedIds)) => val modifiersIds: Seq[Transaction] = requestedIds .map(Algos.encode) - .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } - .flatten + .flatMap(memoryPoolReader.get) logger.debug( s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." @@ -84,8 +80,8 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") } - } + def initBloomFilter: BloomFilter[String] = BloomFilter.create( Funnels.stringFunnel(Charsets.UTF_8), settings.mempool.bloomFilterCapacity, @@ -102,7 +98,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) object MemoryPoolProcessor { - def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) + def props(settings: EncryAppSettings, ntp: NetworkTimeProvider): Props = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter } diff --git a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala b/src/main/scala/encry/view/mempool/MemoryPoolReader.scala new file mode 100644 index 0000000000..470317f68b --- /dev/null +++ b/src/main/scala/encry/view/mempool/MemoryPoolReader.scala @@ -0,0 +1,13 @@ +package encry.view.mempool + +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction + +trait MemoryPoolReader { + def get(elem: String): Option[Transaction] +} + +object MemoryPoolReader { + def apply(pool: MemoryPoolStorage): MemoryPoolReader = (elem: String) => pool.get(elem) + + def empty: MemoryPoolReader = (_: String) => None +} diff --git a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala b/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala index 8f0d68335a..1c6c1dd005 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala @@ -5,9 +5,11 @@ import encry.utils.NetworkTimeProvider import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -final case class MemoryPoolStorage private(transactions: Map[String, Transaction], - settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider) { +final case class MemoryPoolStorage private ( + transactions: Map[String, Transaction], + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider +) { lazy val size: Int = transactions.size @@ -57,13 +59,13 @@ final case class MemoryPoolStorage private(transactions: Map[String, Transaction } def getTransactionsForMiner: (MemoryPoolStorage, Seq[Transaction]) = { - val (transactionsForMiner: Seq[Transaction], _) = transactions - .toIndexedSeq - .sortBy { case (_, tx) => tx.fee } + val (transactionsForMiner: Seq[Transaction], _) = transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } .foldLeft(Seq.empty[Transaction], Set.empty[String]) { case ((validated, inputs), (_, transaction)) => val transactionInputsIds: Set[String] = transaction.inputs.map(input => Algos.encode(input.boxId)).toSet - if (transactionInputsIds.size == transaction.inputs.size && transactionInputsIds.forall(id => !inputs.contains(id))) + if (transactionInputsIds.size == transaction.inputs.size && transactionInputsIds.forall( + id => !inputs.contains(id) + )) (validated :+ transaction, inputs ++ transactionInputsIds) else (validated, inputs) } @@ -81,4 +83,4 @@ object MemoryPoolStorage { def empty(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider): MemoryPoolStorage = MemoryPoolStorage(Map.empty[String, Transaction], settings, networkTimeProvider) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index fe00f24a46..4d36f95cc5 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -1,9 +1,9 @@ package encry.view.mempool import TransactionProto.TransactionProtoMessage -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.BlackList.BanReason.{ CorruptedSerializedBytes, SyntacticallyInvalidTransaction } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer @@ -11,22 +11,20 @@ import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} -import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import encry.view.mempool.TransactionsValidator.InvalidTransaction +import org.encryfoundation.common.modifiers.mempool.transaction.TransactionProtoSerializer import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage -import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } -class TransactionsValidator(settings: EncryAppSettings, - memPool: ActorRef, - networkTimeProvider: NetworkTimeProvider) +class TransactionsValidator(settings: EncryAppSettings, memPool: ActorRef, networkTimeProvider: NetworkTimeProvider) extends Actor with StrictLogging { override def receive(): Receive = { - case DataFromPeer(ModifiersNetworkMessage(data), remote) if data._1 == Transaction.modifierTypeId => - data._2.foreach { + case DataFromPeer(ModifiersNetworkMessage(data), remote) => + data._2.foreach { case (id, bytes) => Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) @@ -39,15 +37,17 @@ class TransactionsValidator(settings: EncryAppSettings, context.parent ! InvalidModifierBytes(id) logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } - } + } } } object TransactionsValidator { - final case class ModifiersForValidating(remote: ConnectedPeer, - typeId: ModifierTypeId, - modifiers: Map[ModifierId, Array[Byte]]) + final case class ModifiersForValidating( + remote: ConnectedPeer, + typeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]] + ) final case class InvalidTransaction(ids: ModifierId) extends AnyVal From c2e858216d53b077d1df6e1a7d848215b6df1a5a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 15:45:03 +0300 Subject: [PATCH 077/119] some fixes --- src/main/scala/encry/view/mempool/IntermediaryMempool.scala | 2 +- src/main/scala/encry/view/mempool/MemoryPool.scala | 2 -- src/test/scala/encry/view/mempool/MemoryPoolTests.scala | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 3e422ae5b8..1872fa92d2 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -23,7 +23,7 @@ class IntermediaryMempool(settings: EncryAppSettings, context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference), name = "mempool") + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), name = "mempool") val txValidator: ActorRef = context.actorOf( diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 47e86b213f..72221153ea 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -105,8 +105,6 @@ class MemoryPool( case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") } - case IsChainSynced(info) => chainSynced = info - case NewTransaction(transaction) => val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index f8d05b8623..efdf12453e 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -63,7 +63,7 @@ class MemoryPoolTests "send transactions to miner" in { val miner1 = TestProbe() val mempool1: TestActorRef[MemoryPool] = - TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, miner1.ref, Some(TestProbe().ref))) + TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, Some(TestProbe().ref), miner1.ref)) val transactions1 = (0 until 4).map { k => val a = coinbaseAt(k) a From b1020ce1d58d365a3d4590201c5b4af5f0f46a78 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 17:46:29 +0300 Subject: [PATCH 078/119] mempool moved --- src/main/resources/TestNetSettings.conf | 2 +- src/main/resources/application.conf | 2 +- src/main/scala/encry/Starter.scala | 14 +++++++----- .../http/routes/TransactionsApiRoute.scala | 2 +- .../api/http/routes/WalletInfoApiRoute.scala | 2 +- .../encry/cli/commands/CreateToken.scala | 2 +- .../scala/encry/cli/commands/Transfer.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 2 +- .../mempool => mpg}/IntermediaryMempool.scala | 22 ++++++++++--------- .../{view/mempool => mpg}/MemoryPool.scala | 6 ++--- .../mempool => mpg}/MemoryPoolProcessor.scala | 6 ++--- .../mempool => mpg}/MemoryPoolReader.scala | 2 +- .../mempool => mpg}/MemoryPoolStorage.scala | 2 +- .../TransactionsValidator.scala | 6 ++--- .../scala/encry/network/DeliveryManager.scala | 2 +- .../DownloadedModifiersValidator.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- .../encry/view/mempool/MemoryPoolTests.scala | 5 +++-- 20 files changed, 46 insertions(+), 41 deletions(-) rename src/main/scala/encry/{view/mempool => mpg}/IntermediaryMempool.scala (78%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPool.scala (98%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolProcessor.scala (95%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolReader.scala (92%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolStorage.scala (99%) rename src/main/scala/encry/{view/mempool => mpg}/TransactionsValidator.scala (94%) diff --git a/src/main/resources/TestNetSettings.conf b/src/main/resources/TestNetSettings.conf index 5c199c0e6f..0dabd402aa 100644 --- a/src/main/resources/TestNetSettings.conf +++ b/src/main/resources/TestNetSettings.conf @@ -145,7 +145,7 @@ delivery-manager-dispatcher { throughput = 1 } mempool-dispatcher { - mailbox-type = "encry.view.mempool.MemoryPool$MemoryPoolPriorityQueue" + mailbox-type = "encry.mpg.MemoryPool$MemoryPoolPriorityQueue" type = Dispatcher executor = "thread-pool-executor" thread-pool-executor.fixed-pool-size = 2 diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 2c97730e8b..d89602ab55 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -218,7 +218,7 @@ delivery-manager-dispatcher { throughput = 1 } mempool-dispatcher { - mailbox-type = "encry.view.mempool.MemoryPool$MemoryPoolPriorityQueue" + mailbox-type = "encry.mpg.MemoryPool$MemoryPoolPriorityQueue" type = Dispatcher executor = "thread-pool-executor" thread-pool-executor.fixed-pool-size = 2 diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 0128ea4043..b216f699ba 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,10 +1,11 @@ package encry import java.net.InetSocketAddress -import akka.actor.{ Actor, ActorRef } + +import akka.actor.{Actor, ActorRef} import akka.http.scaladsl.Http import cats.Functor -import cats.data.{ NonEmptyChain, Validated } +import cats.data.{NonEmptyChain, Validated} import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -15,19 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{ prompt, StartListening } +import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining +import encry.mpg.IntermediaryMempool import encry.network.NetworkRouter import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender -import encry.utils.{ Mnemonic, NetworkTimeProvider } -import encry.view.mempool.IntermediaryMempool +import encry.utils.{Mnemonic, NetworkTimeProvider} import encry.view.wallet.AccountManager + import scala.concurrent.Future import scala.io.StdIn -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, diff --git a/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala b/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala index 4697dec63e..84d1548a59 100644 --- a/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.server.Route import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import encry.settings.RESTApiSettings -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction case class TransactionsApiRoute(dataHolder: ActorRef, memoryPoolRef: ActorRef, settings: RESTApiSettings)( diff --git a/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala b/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala index 17e4cc51d5..49183f99f9 100644 --- a/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala @@ -13,7 +13,7 @@ import encry.modifiers.mempool.TransactionFactory import encry.settings.RESTApiSettings import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import io.circe.syntax._ diff --git a/src/main/scala/encry/cli/commands/CreateToken.scala b/src/main/scala/encry/cli/commands/CreateToken.scala index c1f9524194..127a8c214d 100644 --- a/src/main/scala/encry/cli/commands/CreateToken.scala +++ b/src/main/scala/encry/cli/commands/CreateToken.scala @@ -11,7 +11,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 diff --git a/src/main/scala/encry/cli/commands/Transfer.scala b/src/main/scala/encry/cli/commands/Transfer.scala index 0f75eb828a..8ecefb792b 100644 --- a/src/main/scala/encry/cli/commands/Transfer.scala +++ b/src/main/scala/encry/cli/commands/Transfer.scala @@ -13,7 +13,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b2b45cd2f4..9001fe3cd2 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -23,7 +23,7 @@ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.TransactionsForMiner +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.utils.implicits.UTXO._ import encry.view.wallet.EncryWallet diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala similarity index 78% rename from src/main/scala/encry/view/mempool/IntermediaryMempool.scala rename to src/main/scala/encry/mpg/IntermediaryMempool.scala index 1872fa92d2..2903da3c3f 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging @@ -9,21 +9,23 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForTxHandling import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ RolledBackTransactions, TransactionProcessing } -import encry.view.mempool.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } - -class IntermediaryMempool(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - influxReference: Option[ActorRef], - networkRouter: ActorRef) - extends Actor +import encry.mpg.MemoryPool._ +import encry.mpg.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } + +class IntermediaryMempool( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + networkRouter: ActorRef +) extends Actor with StrictLogging { val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), name = "mempool") + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), + name = "mempool") val txValidator: ActorRef = context.actorOf( diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala similarity index 98% rename from src/main/scala/encry/view/mempool/MemoryPool.scala rename to src/main/scala/encry/mpg/MemoryPool.scala index 72221153ea..2573685368 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, ActorRef, ActorSystem, Props } import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } @@ -7,18 +7,18 @@ import com.google.common.base.Charsets import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging +import encry.mpg.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions -import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.mpg.MemoryPool._ import scala.collection.IndexedSeq diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/mpg/MemoryPoolProcessor.scala similarity index 95% rename from src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala rename to src/main/scala/encry/mpg/MemoryPoolProcessor.scala index b86b30fa20..61e1173538 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/mpg/MemoryPoolProcessor.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, Props } import com.google.common.base.Charsets @@ -9,12 +9,12 @@ import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromL import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ TransactionProcessing, UpdateMempoolReader } -import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.mpg.MemoryPool._ +import encry.mpg.MemoryPoolProcessor.CleanupBloomFilter import scala.collection.IndexedSeq diff --git a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala b/src/main/scala/encry/mpg/MemoryPoolReader.scala similarity index 92% rename from src/main/scala/encry/view/mempool/MemoryPoolReader.scala rename to src/main/scala/encry/mpg/MemoryPoolReader.scala index 470317f68b..1e51d26892 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala +++ b/src/main/scala/encry/mpg/MemoryPoolReader.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala b/src/main/scala/encry/mpg/MemoryPoolStorage.scala similarity index 99% rename from src/main/scala/encry/view/mempool/MemoryPoolStorage.scala rename to src/main/scala/encry/mpg/MemoryPoolStorage.scala index 1c6c1dd005..bcf425c729 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala +++ b/src/main/scala/encry/mpg/MemoryPoolStorage.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/mpg/TransactionsValidator.scala similarity index 94% rename from src/main/scala/encry/view/mempool/TransactionsValidator.scala rename to src/main/scala/encry/mpg/TransactionsValidator.scala index 4d36f95cc5..ca583e480d 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/mpg/TransactionsValidator.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import TransactionProto.TransactionProtoMessage import akka.actor.{ Actor, ActorRef, Props } @@ -10,11 +10,11 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.InvalidTransaction import org.encryfoundation.common.modifiers.mempool.transaction.TransactionProtoSerializer import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import encry.mpg.MemoryPool._ +import encry.mpg.TransactionsValidator.InvalidTransaction import scala.util.{ Failure, Success, Try } diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 64ade81e21..47c475f688 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -27,7 +27,7 @@ import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo._ diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 8732b1d7eb..d907297913 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -14,7 +14,7 @@ import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index e3d4b4bbbb..718adb3148 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -10,6 +10,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -22,7 +23,6 @@ import encry.utils.Utils._ import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.NodeViewErrors.ModifierApplyError import encry.view.history.History -import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} import org.encryfoundation.common.modifiers.history._ diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index d789ef0ed3..cb26c80753 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -34,7 +34,7 @@ import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.mpg.MemoryPool._ import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestChunkMessage, diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f628a3f571..30f2126fbe 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -27,7 +27,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index efdf12453e..76fc47ff40 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -4,9 +4,10 @@ import akka.actor.ActorSystem import akka.testkit.{ TestActorRef, TestProbe } import com.typesafe.scalalogging.StrictLogging import encry.modifiers.InstanceFactory -import encry.settings.{ EncryAppSettings, TestNetSettings } +import encry.mpg.MemoryPool._ +import encry.mpg.{ MemoryPool, MemoryPoolStorage } +import encry.settings.TestNetSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ NewTransaction, TransactionsForMiner } import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scala.concurrent.duration._ From fa9a626371d7a80babbd781b3ac0d645fb7f12df Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 17:57:28 +0300 Subject: [PATCH 079/119] add dataholder --- src/main/scala/encry/Starter.scala | 4 ++-- src/main/scala/encry/network/DM.scala | 6 +++--- src/main/scala/encry/network/MessageBuilder.scala | 1 + src/main/scala/encry/nvg/IntermediaryNVH.scala | 12 ++++++++---- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index b216f699ba..65ce379acf 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -406,7 +406,7 @@ class Starter(settings: EncryAppSettings, context.system .actorOf(StatsSender.props(influxSettings, newSettings.network, newSettings.constants), "statsSender") } - lazy val dataHolderForApi = + val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) @@ -422,7 +422,7 @@ class Starter(settings: EncryAppSettings, IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) val nvhRouter: ActorRef = context.system.actorOf( - IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool) + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool, dataHolderForApi) ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index b766d63723..841b48cf5e 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -38,7 +38,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! ModifierFromNetwork(source, data._1, id, bytes) expectedModifiers -= toKey(id) receivedModifier += toKey(id) - } else logger.info("Receive spam!") + } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> expectedModifiers += toKey(modId) @@ -67,8 +67,8 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case IsRequested(modIds) => //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") sender ! RequestStatus( - modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), - modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + requested = modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + notRequested = modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) ) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 544af268ea..3adeba3341 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -33,6 +33,7 @@ case class MessageBuilder(peersKeeper: ActorRef, (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + logger.info(s"Send req for mods: ${status.notRequested.map(Algos.encode).mkString(",")} to peer ${peer.socketAddress}") logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index cb26c80753..18dac26955 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -51,7 +51,8 @@ class IntermediaryNVH( intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, influxRef: Option[ActorRef], - mempoolRef: ActorRef + mempoolRef: ActorRef, + dataHolderRef: ActorRef ) extends Actor with StrictLogging { @@ -109,6 +110,7 @@ class IntermediaryNVH( case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg + dataHolderRef ! msg case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg @@ -126,7 +128,8 @@ class IntermediaryNVH( case msg @ RolledBackTransactions(_) => mempoolRef ! msg case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg + case msg: UpdatedHistory => dataHolderRef ! msg case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => @@ -144,6 +147,7 @@ object IntermediaryNVH { intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, influxRef: Option[ActorRef], - mempoolRef: ActorRef - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef)) + mempoolRef: ActorRef, + dataHolderRef: ActorRef + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 30f2126fbe..1dc865246d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -283,6 +283,7 @@ class NodeViewHolder( if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! UpdatedHistory(history) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) From e3b3326fac0b6a79e18394a8ce68543e0318ea67 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 18:09:31 +0300 Subject: [PATCH 080/119] use history reader instead of history --- .../encry/api/http/DataHolderForApi.scala | 12 ++++----- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 - .../encry/view/history/HistoryReader.scala | 26 +++++++++++++++++-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 935b592d8d..037c3ab2a6 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,11 +21,11 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.NodeViewHolder.NodeViewChange +import encry.nvg.NodeViewHolder.{NodeViewChange, UpdateHistoryReader} import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView -import encry.view.history.History +import encry.view.history.{History, HistoryReader} import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 @@ -55,7 +55,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) val launchTimeFuture: Future[NetworkTime.Time] = ntp.time() def awaitNVHRef: Receive = { - case UpdatedHistory(history) => + case UpdateHistoryReader(history) => unstashAll() context.become(workingCycle(nvhRef = sender(), history = Some(history))) case PassForStorage(_) => @@ -66,7 +66,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) def workingCycle(nvhRef: ActorRef, blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))] = Seq.empty, connectedPeers: Seq[ConnectedPeer] = Seq.empty, - history: Option[History] = None, + history: Option[HistoryReader] = None, state: Option[UtxoStateReader] = None, transactionsOnMinerActor: Int = 0, minerStatus: MinerStatus = MinerStatus(isMining = false, None), @@ -122,7 +122,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) connectedPeersCollection) ) - case ChangedHistory(reader: History) => + case UpdateHistoryReader(reader: HistoryReader) => context.become( workingCycle(nvhRef, blackList, @@ -333,7 +333,7 @@ object DataHolderForApi { //scalastyle:ignore final case class GetLastHeaderIdAtHeightHelper(i: Int) - final case class Readers(h: Option[History], s: Option[UtxoStateReader]) + final case class Readers(h: Option[HistoryReader], s: Option[UtxoStateReader]) final case class PassForStorage(password: String) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 18dac26955..5d4127fe0c 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -129,7 +129,7 @@ class IntermediaryNVH( case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg - case msg: UpdatedHistory => dataHolderRef ! msg + case msg: UpdateHistoryReader => dataHolderRef ! msg case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 1dc865246d..30f2126fbe 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -283,7 +283,6 @@ class NodeViewHolder( if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) - context.parent ! UpdatedHistory(history) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 8e3deb193e..64f9c9edb7 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,7 +1,9 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } -import org.encryfoundation.common.modifiers.history.Header +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.modifiers.history.HeaderChain +import io.iohk.iodb.ByteArrayWrapper +import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -19,16 +21,26 @@ trait HistoryReader { def compare(si: SyncInfo): HistoryComparisonResult + def getHeaderById(id: ModifierId): Option[Header] + + def getBlockByHeaderId(id: ModifierId): Option[Block] + + def getBlockByHeader(header: Header): Option[Block] + var isFullChainSynced: Boolean var isHeadersChainSyncedVar: Boolean = false def isModifierDefined(id: ModifierId): Boolean + def headerIdsAtHeight(height: Int): List[ModifierId] + def modifierBytesById(id: ModifierId): Option[Array[Byte]] def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + def lastHeaders(count: Int): HeaderChain + def syncInfo: SyncInfo def isFastSyncInProcess: Boolean @@ -47,6 +59,11 @@ object HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty def syncInfo: SyncInfo = SyncInfo(Seq.empty) def isFastSyncInProcess: Boolean = false + def getHeaderById(id: ModifierId): Option[Header] = None + def getBlockByHeaderId(id: ModifierId): Option[Block] = None + def getBlockByHeader(header: Header): Option[Block] = None + def headerIdsAtHeight(height: Int): List[ModifierId] = List.empty[ModifierId] + def lastHeaders(count: Int): HeaderChain = HeaderChain.empty } def apply(history: History): HistoryReader = new HistoryReader { @@ -61,5 +78,10 @@ object HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) def syncInfo: SyncInfo = history.syncInfo def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal + def getHeaderById(id: ModifierId): Option[Header] = history.getHeaderById(id) + def headerIdsAtHeight(height: Int): List[ModifierId] = history.headerIdsAtHeight(height).toList + def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) + def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) + def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) } } From 159b5a83a2ddabc291afc564e9e8a9f5542050b2 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 18:39:39 +0300 Subject: [PATCH 081/119] send info about peers from network router to data holder --- src/main/scala/encry/Starter.scala | 2 +- .../encry/api/http/DataHolderForApi.scala | 21 +++++++------------ .../encry/api/http/routes/PeersApiRoute.scala | 6 +++--- .../scala/encry/network/NetworkRouter.scala | 8 +++++-- src/main/scala/encry/network/PK.scala | 20 ++++++++++++++++-- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 65ce379acf..bebdd2fc7c 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -413,7 +413,7 @@ class Starter(settings: EncryAppSettings, val networkRouter = context.system.actorOf( NetworkRouter - .props(networkSettings, settings.blackList) + .props(networkSettings, settings.blackList, dataHolderForApi) .withDispatcher("nvsh-dispatcher"), "networkRouter" ) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 037c3ab2a6..4382cab089 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -19,7 +19,7 @@ import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, ConnectionType} import encry.network.PeersKeeper.BanPeerFromAPI import encry.nvg.NodeViewHolder.{NodeViewChange, UpdateHistoryReader} import encry.settings.EncryAppSettings @@ -64,14 +64,14 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) } def workingCycle(nvhRef: ActorRef, - blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))] = Seq.empty, - connectedPeers: Seq[ConnectedPeer] = Seq.empty, + blackList: List[(InetAddress, (BanReason, BanTime, BanType))] = List.empty, + connectedPeers: List[(InetSocketAddress, String, ConnectionType)] = List.empty, history: Option[HistoryReader] = None, state: Option[UtxoStateReader] = None, transactionsOnMinerActor: Int = 0, minerStatus: MinerStatus = MinerStatus(isMining = false, None), blockInfo: BlockAndHeaderInfo = BlockAndHeaderInfo(None, None), - allPeers: Seq[InetSocketAddress] = Seq.empty, + allPeers: List[InetSocketAddress] = List.empty, connectedPeersCollection: ConnectedPeersCollection = ConnectedPeersCollection()): Receive = { case UpdatingTransactionsNumberForApi(qty) => @@ -230,12 +230,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) case GetBannedPeersHelper => sender() ! blackList case GetConnectedPeersHelper => sender() ! connectedPeers - .map( - peer => - PeerInfoResponse(peer.socketAddress.toString, - Some(peer.handshake.nodeName), - Some(peer.direction.toString))) - + .map(peer => PeerInfoResponse(peer._1.toString, peer._2, peer._3.toString)) case GetLastHeaderIdAtHeightHelper(i) => sender() ! history.toList.flatMap(_.headerIdsAtHeight(i).map(Algos.encode)) @@ -317,9 +312,9 @@ object DataHolderForApi { //scalastyle:ignore final case class UpdatingMinerStatus(minerStatus: MinerStatus) extends AnyVal - final case class UpdatingPeersInfo(allPeers: Seq[InetSocketAddress], - connectedPeers: Seq[ConnectedPeer], - blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))]) + final case class UpdatingPeersInfo(allPeers: List[InetSocketAddress], + connectedPeers: List[(InetSocketAddress, String, ConnectionType)], + blackList: List[(InetAddress, (BanReason, BanTime, BanType))]) final case class BlockAndHeaderInfo(header: Option[Header], block: Option[Block]) diff --git a/src/main/scala/encry/api/http/routes/PeersApiRoute.scala b/src/main/scala/encry/api/http/routes/PeersApiRoute.scala index 4130e69ac5..d0cf1af184 100755 --- a/src/main/scala/encry/api/http/routes/PeersApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/PeersApiRoute.scala @@ -76,14 +76,14 @@ case class PeersApiRoute(override val settings: RESTApiSettings, dataHolder: Act object PeersApiRoute { - case class PeerInfoResponse(address: String, name: Option[String], connectionType: Option[String]) + case class PeerInfoResponse(address: String, name: String, connectionType: String) object PeerInfoResponse { def fromAddressAndInfo(address: InetSocketAddress, peerInfo: PeerInfo): PeerInfoResponse = PeerInfoResponse( address.toString, - Some(peerInfo.connectedPeer.toString), - Some(peerInfo.connectionType.toString) + peerInfo.connectedPeer.toString, + peerInfo.connectionType.toString ) } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 9ceb8a9576..419ab5bab3 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -7,6 +7,7 @@ import akka.io.Tcp.{Bind, Bound, CommandFailed, Connect, Connected} import akka.io.{IO, Tcp} import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent @@ -28,7 +29,8 @@ import scala.concurrent.duration._ import scala.util.Random class NetworkRouter(settings: NetworkSettings, - blackListSettings: BlackListSettings) extends Actor with StrictLogging { + blackListSettings: BlackListSettings, + dataHolderRef: ActorRef) extends Actor with StrictLogging { import context.system import context.dispatcher @@ -78,6 +80,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork if msg.modTypeId != Transaction.modifierTypeId => handlerForMods ! msg case msg: ModifierFromNetwork => txsHandler ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg + case msg: UpdatingPeersInfo => dataHolderRef ! msg case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), @@ -138,5 +141,6 @@ object NetworkRouter { case object RegisterForTxHandling def props(settings: NetworkSettings, - blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) + blackListSettings: BlackListSettings, + dataHolderRef: ActorRef): Props = Props(new NetworkRouter(settings, blackListSettings, dataHolderRef)) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 0ab62b5224..8b175cfcd2 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,8 +2,9 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, Props} +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.PeerInfo @@ -12,17 +13,20 @@ import encry.network.Messages.MessageToNetwork.SendPeers import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection -import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} +import scala.concurrent.duration._ import scala.util.{Random, Try} class PK(networkSettings: NetworkSettings, blacklistSettings: BlackListSettings) extends Actor with StrictLogging { + import context.dispatcher + val connectWithOnlyKnownPeers: Boolean = networkSettings.connectOnlyWithKnownPeers.getOrElse(true) var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() @@ -44,6 +48,15 @@ class PK(networkSettings: NetworkSettings, PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" ), self) + context.system.scheduler.schedule(5.seconds, 5.seconds) { + context.parent ! UpdatingPeersInfo( + peersForConnection.keys.toList, + connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => + (peer.socketAddress, peer.handshake.nodeName, peer.direction) + ).toList, + blackList.getAll.toList + ) + } } override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { @@ -182,6 +195,9 @@ class PK(networkSettings: NetworkSettings, networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) + + def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true + def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer } object PK { From caca490355c7a8d14071e6593bf2dadc7fb111a3 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:45:27 +0300 Subject: [PATCH 082/119] stop mining during rollback, send txs for miner by ask --- src/main/scala/encry/local/miner/Miner.scala | 17 ++++++++------- .../scala/encry/mpg/IntermediaryMempool.scala | 1 + src/main/scala/encry/mpg/MemoryPool.scala | 12 +++-------- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 1 - src/main/scala/encry/nvg/NodeViewHolder.scala | 21 ++++++++++--------- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 9001fe3cd2..b8531104b6 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -5,6 +5,7 @@ import java.util.Date import akka.actor.{Actor, ActorRef, Props} import akka.util.Timeout +import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.EncryApp._ @@ -63,7 +64,7 @@ class Miner(dataHolder: ActorRef, var transactionsPool: IndexedSeq[Transaction] = IndexedSeq.empty[Transaction] override def preStart(): Unit = { - context.system.eventStream.subscribe(self, classOf[ClIMiner]) + context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) context.system.scheduler.schedule(5.seconds, 5.seconds)( @@ -228,7 +229,7 @@ class Miner(dataHolder: ActorRef, } def produceCandidate(): Unit = { - val lambda = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => + def lambda(txs: List[Transaction]) = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => { val producingStartTime: Time = System.currentTimeMillis() startTime = producingStartTime @@ -252,19 +253,21 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda) + (mempool ? SendTransactionsToMiner).mapTo[List[Transaction]].foreach { txs => + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs)) + } } } object Miner { - sealed trait ClIMiner + sealed trait MinerMiningCommands - case object DisableMining extends ClIMiner + case object DisableMining extends MinerMiningCommands - case object EnableMining extends ClIMiner + case object EnableMining extends MinerMiningCommands - case object StartMining extends ClIMiner + case object StartMining extends MinerMiningCommands case class MinedBlock(block: Block, workerIdx: Int) diff --git a/src/main/scala/encry/mpg/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala index 2903da3c3f..51d4c450a0 100644 --- a/src/main/scala/encry/mpg/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -44,6 +44,7 @@ class IntermediaryMempool( case msg: RequestFromLocal => networkRouter ! msg case msg: ModifierFromNetwork => txValidator ! msg case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ SendTransactionsToMiner => memoryPool.forward(msg) case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } diff --git a/src/main/scala/encry/mpg/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala index 2573685368..5bf935e1db 100644 --- a/src/main/scala/encry/mpg/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -48,12 +48,6 @@ class MemoryPool( self, RemoveExpiredFromPool ) - context.system.scheduler.schedule( - settings.mempool.txSendingInterval, - settings.mempool.txSendingInterval, - self, - SendTransactionsToMiner - ) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } @@ -181,7 +175,7 @@ class MemoryPool( memoryPool.getTransactionsForMiner memoryPool = newMemoryPool mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) - context.parent ! TransactionsForMiner(transactionsForMiner) + sender() ! TransactionsForMiner(transactionsForMiner) logger.debug( s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}" @@ -220,9 +214,9 @@ object MemoryPool { final case class NewTransaction(tx: Transaction) extends AnyVal - final case class RolledBackTransactions(txs: IndexedSeq[Transaction]) extends AnyVal + final case class RolledBackTransactions(txs: IndexedSeq[Transaction]) - final case class TransactionsForMiner(txs: Seq[Transaction]) extends AnyVal + final case class TransactionsForMiner(txs: Seq[Transaction]) case object SendTransactionsToMiner diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 718adb3148..45fd0cde77 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -9,7 +9,7 @@ import akka.util.Timeout import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.local.miner.Miner.{MinerMiningCommands, DisableMining, StartMining} import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 5d4127fe0c..b6b6b65a1f 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -126,7 +126,6 @@ class IntermediaryNVH( case msg @ HeaderChainIsSynced => case msg @ FullBlockChainIsSynced => mempoolRef ! msg case msg @ RolledBackTransactions(_) => mempoolRef ! msg - case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg case msg: UpdateHistoryReader => dataHolderRef ! msg diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 30f2126fbe..c3e88618b3 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, EnableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.nvg.fast.sync.SnapshotProcessor._ @@ -26,23 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -224,7 +224,7 @@ class NodeViewHolder( val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.parent ! DisableMining + context.system.eventStream.publish(DisableMining) state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) } else Success(state) -> IndexedSeq.empty @@ -233,6 +233,7 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) + if (settings.node.mining && nodeView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => From e0bc8cd111a405102c8382cbe936c0001a0028d5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:49:32 +0300 Subject: [PATCH 083/119] ec-fix --- src/main/scala/encry/local/miner/Miner.scala | 4 ++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b8531104b6..c6e1974fcd 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -253,8 +253,8 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - (mempool ? SendTransactionsToMiner).mapTo[List[Transaction]].foreach { txs => - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs)) + (mempool ? SendTransactionsToMiner).mapTo[TransactionsForMiner].foreach { txs => + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs.txs.toList)) } } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index c3e88618b3..17b6488ca8 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -233,7 +233,7 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) - if (settings.node.mining && nodeView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) + if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) context.system.eventStream.publish(EnableMining) val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => From 77c9107c7d8bc0ed8b8982934280633c5a4a3947 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:54:48 +0300 Subject: [PATCH 084/119] remove mempool buffer at miner --- src/main/scala/encry/local/miner/Miner.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index c6e1974fcd..7a59b31a61 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -61,18 +61,13 @@ class Miner(dataHolder: ActorRef, val numberOfWorkers: Int = settings.node.numberOfMiningWorkers val powScheme: EquihashPowScheme = EquihashPowScheme(TestNetConstants.n, TestNetConstants.k, TestNetConstants.Version, TestNetConstants.PreGenesisHeight, TestNetConstants.MaxTarget) - var transactionsPool: IndexedSeq[Transaction] = IndexedSeq.empty[Transaction] override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) - context.system.scheduler.schedule(5.seconds, 5.seconds)( - influx.foreach(_ ! InfoAboutTransactionsFromMiner(transactionsPool.size)) - ) context.system.scheduler.schedule(5.seconds, 5.seconds) { logger.info(s"data holder: $dataHolder. Context: $context") - dataHolder ! UpdatingTransactionsNumberForApi(transactionsPool.length) dataHolder ! UpdatingMinerStatus(MinerStatus(context.children.nonEmpty && candidateOpt.nonEmpty, candidateOpt)) } } @@ -104,7 +99,6 @@ class Miner(dataHolder: ActorRef, logger.info("Candidate is empty! Producing new candidate!") produceCandidate() } - case TransactionsForMiner(txs) => transactionsPool = transactionsPool ++ txs case StartMining => logger.info("Can't start mining because of chain is not synced!") case DisableMining if context.children.nonEmpty => println(s"Miner -> Disable mining context.children.nonEmpty") @@ -182,10 +176,11 @@ class Miner(dataHolder: ActorRef, } def createCandidate(view: CurrentView[History, UtxoState, EncryWallet], + txsFromMempool: List[Transaction], bestHeaderOpt: Option[Header]): CandidateBlock = { val height: Height = Height @@ (bestHeaderOpt.map(_.height).getOrElse(TestNetConstants.PreGenesisHeight) + 1) val timestamp: Time = timeProvider.estimatedTime - val txsU: IndexedSeq[Transaction] = transactionsPool.filter(view.state.validate(_, timestamp, height).isRight).distinct + val txsU: List[Transaction] = txsFromMempool.filter(view.state.validate(_, timestamp, height).isRight).distinct val filteredTxsWithoutDuplicateInputs = txsU.foldLeft(List.empty[String], IndexedSeq.empty[Transaction]) { case ((usedInputsIds, acc), tx) => if (tx.inputs.forall(input => !usedInputsIds.contains(Algos.encode(input.boxId)))) { @@ -224,7 +219,6 @@ class Miner(dataHolder: ActorRef, logger.info(s"Sending candidate block with ${candidate.transactions.length - 1} transactions " + s"and 1 coinbase for height $height.") - transactionsPool = IndexedSeq.empty[Transaction] candidate } @@ -248,7 +242,7 @@ class Miner(dataHolder: ActorRef, logger.info("Going to calculate last block:") val envelope: CandidateEnvelope = CandidateEnvelope - .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) + .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt)) envelope } else CandidateEnvelope.empty candidate From 2b1c425f786e0e8a931fd7bbc36a2c42bfd404e7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 13 Mar 2020 12:30:00 +0300 Subject: [PATCH 085/119] mempool keeps txs till ssm --- .../scala/encry/mpg/IntermediaryMempool.scala | 22 ++++---- src/main/scala/encry/mpg/MemoryPool.scala | 23 ++------ .../scala/encry/mpg/MemoryPoolStorage.scala | 10 ++-- .../scala/encry/nvg/IntermediaryNVH.scala | 53 ++++++++----------- src/main/scala/encry/nvg/NodeViewHolder.scala | 28 +++++----- .../encry/view/mempool/MemoryPoolTests.scala | 3 +- 6 files changed, 59 insertions(+), 80 deletions(-) diff --git a/src/main/scala/encry/mpg/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala index 51d4c450a0..c7ed461a0b 100644 --- a/src/main/scala/encry/mpg/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -11,6 +11,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.mpg.MemoryPool._ import encry.mpg.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier class IntermediaryMempool( settings: EncryAppSettings, @@ -36,16 +37,17 @@ class IntermediaryMempool( override def preStart(): Unit = networkRouter ! RegisterForTxHandling override def receive(): Receive = { - case msg: InvalidTransaction => networkRouter ! msg - case msg: BanPeer => networkRouter ! msg - case msg: RolledBackTransactions => memoryPool ! msg - case msg: ModifiersForValidating => memoryPool ! msg - case msg: DataFromPeer => mempoolProcessor ! msg - case msg: RequestFromLocal => networkRouter ! msg - case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg - case msg @ SendTransactionsToMiner => memoryPool.forward(msg) - case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ SendTransactionsToMiner => memoryPool.forward(msg) + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg @ SemanticallySuccessfulModifier(_) => memoryPool ! msg } } diff --git a/src/main/scala/encry/mpg/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala index 5bf935e1db..06d645ab0d 100644 --- a/src/main/scala/encry/mpg/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -2,12 +2,11 @@ package encry.mpg import akka.actor.{ Actor, ActorRef, ActorSystem, Props } import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } -import cats.syntax.either._ import com.google.common.base.Charsets import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.mpg.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions +import encry.mpg.MemoryPool._ import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } @@ -18,7 +17,6 @@ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import encry.mpg.MemoryPool._ import scala.collection.IndexedSeq @@ -48,7 +46,6 @@ class MemoryPool( self, RemoveExpiredFromPool ) - context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } override def receive: Receive = continueProcessing(currentNumberOfProcessedTransactions = 0) @@ -153,14 +150,13 @@ class MemoryPool( } def auxiliaryReceive(state: MemoryPoolStateType): Receive = { - case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Block.modifierTypeId => + case SemanticallySuccessfulModifier(modifier: Block) => logger.debug( s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + s"Transit to a transactionsProcessor state." ) - if (state == NotProcessingNewTransactions) - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) canProcessTransactions = true + memoryPool = memoryPool.compareWithMod(modifier) context.parent ! TransactionProcessing(canProcessTransactions) context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) @@ -171,9 +167,7 @@ class MemoryPool( ) case SendTransactionsToMiner => - val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = - memoryPool.getTransactionsForMiner - memoryPool = newMemoryPool + val transactionsForMiner: Seq[Transaction] = memoryPool.getTransactionsForMiner mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) sender() ! TransactionsForMiner(transactionsForMiner) logger.debug( @@ -185,15 +179,6 @@ class MemoryPool( memoryPool = memoryPool.filter(memoryPool.isExpired) logger.debug(s"MemoryPool got RemoveExpiredFromPool message. After cleaning pool size is: ${memoryPool.size}.") -// case RequestModifiersForTransactions(remote, ids) => -// val modifiersIds: Seq[Transaction] = ids -// .map(Algos.encode) -// .collect { case if memorascyPool.contains(id) => memoryPool.get(id) } -// .flatten -// sender() ! RequestedModifiersForRemote(remote, modifiersIds) -// logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + -// s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") - case message => logger.debug(s"MemoryPool got unhandled message $message.") } diff --git a/src/main/scala/encry/mpg/MemoryPoolStorage.scala b/src/main/scala/encry/mpg/MemoryPoolStorage.scala index bcf425c729..5c561f879a 100644 --- a/src/main/scala/encry/mpg/MemoryPoolStorage.scala +++ b/src/main/scala/encry/mpg/MemoryPoolStorage.scala @@ -2,6 +2,7 @@ package encry.mpg import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos @@ -58,8 +59,8 @@ final case class MemoryPoolStorage private ( } } - def getTransactionsForMiner: (MemoryPoolStorage, Seq[Transaction]) = { - val (transactionsForMiner: Seq[Transaction], _) = transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } + def getTransactionsForMiner: Seq[Transaction] = + transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } .foldLeft(Seq.empty[Transaction], Set.empty[String]) { case ((validated, inputs), (_, transaction)) => val transactionInputsIds: Set[String] = transaction.inputs.map(input => Algos.encode(input.boxId)).toSet @@ -69,8 +70,9 @@ final case class MemoryPoolStorage private ( (validated :+ transaction, inputs ++ transactionInputsIds) else (validated, inputs) } - (removeSeveral(transactionsForMiner.map(_.encodedId)), transactionsForMiner) - } + ._1 + + def compareWithMod(block: Block): MemoryPoolStorage = removeSeveral(block.payload.txs.map(_.encodedId)) def isValid: Transaction => Boolean = tx => tx.semanticValidity.isSuccess && !contains(tx.encodedId) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index b6b6b65a1f..4bcf66def2 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -5,7 +5,8 @@ import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } +import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } @@ -13,37 +14,20 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandlin import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.NodeViewHolder.{ - GetDataFromCurrentView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} import encry.nvg.fast.sync.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } -import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader -import encry.mpg.MemoryPool._ -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -124,18 +108,24 @@ class IntermediaryNVH( case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => mempoolRef ! msg - case msg @ RolledBackTransactions(_) => mempoolRef ! msg - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg - case msg: UpdateHistoryReader => dataHolderRef ! msg - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => + case msg @ DisableMining => context.system.eventStream.publish(msg) + case msg @ EnableMining => context.system.eventStream.publish(msg) + case msg @ FullBlockChainIsSynced => + context.system.eventStream.publish(msg) + mempoolRef ! msg + case msg @ RolledBackTransactions(_) => mempoolRef ! msg + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg + case msg: UpdateHistoryReader => dataHolderRef ! msg + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + context.system.eventStream.publish(msg) intermediaryNetwork ! msg networkMessagesProcessor ! msg + mempoolRef ! msg case msg @ SemanticallyFailedModification(_, _) => intermediaryNetwork ! msg } } @@ -148,5 +138,6 @@ object IntermediaryNVH { influxRef: Option[ActorRef], mempoolRef: ActorRef, dataHolderRef: ActorRef - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) + ): Props = + Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 17b6488ca8..7d1d758d3e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, EnableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.nvg.fast.sync.SnapshotProcessor._ @@ -26,23 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -224,7 +224,7 @@ class NodeViewHolder( val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.system.eventStream.publish(DisableMining) + context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) } else Success(state) -> IndexedSeq.empty @@ -233,7 +233,8 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) - if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) context.system.eventStream.publish(EnableMining) + if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) + context.parent ! EnableMining val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => @@ -283,7 +284,7 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! SemanticallySuccessfulModifier(modToApply) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) @@ -355,14 +356,13 @@ class NodeViewHolder( logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() context.parent ! FullBlockChainIsSynced - context.system.eventStream.publish(FullBlockChainIsSynced) } updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + context.parent ! SemanticallySuccessfulModifier(modifier) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e: Throwable) => diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index 76fc47ff40..a28c915c3c 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -54,8 +54,7 @@ class MemoryPoolTests val mempool = MemoryPoolStorage.empty(testNetSettings, timeProvider) val transactions = (0 until 10).map(k => coinbaseAt(k)) val (newMempool, _) = mempool.validateTransactions(transactions) - val (uPool, txs) = newMempool.getTransactionsForMiner - uPool.size shouldBe 0 + val txs = newMempool.getTransactionsForMiner txs.map(_.encodedId).forall(transactions.map(_.encodedId).contains) shouldBe true transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } From 96e90f9e903b6036d0113be73df0dce44d3db55b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 13 Mar 2020 15:33:07 +0300 Subject: [PATCH 086/119] request for headers fixed --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8415387d83..200f5a008c 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -99,7 +99,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => + case RequestModifiersNetworkMessage((typeId, requestedIds)) => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap( (id: ModifierId) => From 96b6c170986573df2e8c76c53a29e82a7f1d99ea Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 11:39:33 +0300 Subject: [PATCH 087/119] change messageBuilder naming --- src/main/scala/encry/network/NetworkRouter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 419ab5bab3..261c6dc881 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -84,7 +84,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), - s"messageBuilder${Random.nextInt()}" + s"messageBuilder${System.currentTimeMillis()}" ) ! msg } From b3eae66aff0d3ae558b1da795bc6e4a27d73eda5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 11:58:53 +0300 Subject: [PATCH 088/119] ec fix --- src/main/scala/encry/network/NetworkRouter.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 261c6dc881..2fd32463a6 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -23,10 +23,12 @@ import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scorex.utils.Random import scala.concurrent.duration._ -import scala.util.Random +import scala.util.{Random => SRand} class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings, @@ -84,7 +86,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), - s"messageBuilder${System.currentTimeMillis()}" + s"messageBuilder${Algos.encode(Random.randomBytes()) ++ SRand.nextLong().toString}" ) ! msg } From 3a97b20ed4684249c316fc7d6b2328419d7588db Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 12:09:59 +0300 Subject: [PATCH 089/119] todo added --- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- .../scala/encry/view/history/HistoryPayloadsProcessor.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 7d1d758d3e..7ecff66a98 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -123,7 +123,7 @@ class NodeViewHolder( FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settings + override val settings: EncryAppSettings = settings //todo incorrect override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val timeProvider: NetworkTimeProvider = ntp override val historyStorage: HistoryStorage = nodeView.history.historyStorage diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index ecdc4be798..071e547cf2 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -64,7 +64,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { (fullBlock.header.height > bestHeaderHeight) || ( (fullBlock.header.height == bestHeaderHeight) && scoreOf(fullBlock.id) - .flatMap(fbScore => getBestHeaderId.flatMap(id => scoreOf(id).map(_ < fbScore))) + .flatMap(fbScore => getBestHeaderId.flatMap(scoreOf(_).map(_ < fbScore))) .getOrElse(false) ) val updatedHeadersAtHeightIds = From fb00a55b0baaa29dd8fc39dc5f6a73840663012d Mon Sep 17 00:00:00 2001 From: Lior Date: Mon, 16 Mar 2020 14:20:03 +0300 Subject: [PATCH 090/119] test removed --- .../encry/view/mempool/MemoryPoolTests.scala | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index a28c915c3c..04e5b67301 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -59,19 +59,4 @@ class MemoryPoolTests transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } } - "Mempool actor" should { - "send transactions to miner" in { - val miner1 = TestProbe() - val mempool1: TestActorRef[MemoryPool] = - TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, Some(TestProbe().ref), miner1.ref)) - val transactions1 = (0 until 4).map { k => - val a = coinbaseAt(k) - a - } - transactions1.foreach(mempool1 ! NewTransaction(_)) - mempool1.underlyingActor.memoryPool.size shouldBe 4 - logger.info(s"generated: ${transactions1.map(_.encodedId)}") - miner1.expectMsg(20.seconds, TransactionsForMiner(transactions1)) - } - } } From 27e624072ad07e6104bb0c720d4530c3a2dc1c2a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 14:41:31 +0300 Subject: [PATCH 091/119] fixed modifiers propagation --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 200f5a008c..8577e16d38 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,9 +1,9 @@ package encry.nvg import akka.actor.{ Actor, Cancellable, Props } +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } -import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto @@ -17,7 +17,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, - ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } @@ -108,7 +107,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St .map(id -> _) ) .toMap - if (modifiersFromCache.nonEmpty) context.parent ! ModifiersNetworkMessage(typeId -> modifiersFromCache) + if (modifiersFromCache.nonEmpty) context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList typeId match { From 944ec15416e3314208f6eb0240e95390e7511b8c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 14:42:24 +0300 Subject: [PATCH 092/119] fixed modifiers propagation --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8577e16d38..13761962ca 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -23,7 +23,6 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId - import scala.concurrent.duration._ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { From fc96df1119d11ac531356121e2e2b56ed6ec1ed2 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 16:02:39 +0300 Subject: [PATCH 093/119] add tests --- src/main/scala/encry/network/DM.scala | 2 +- .../DeliveryManagerTests/DMUtils.scala | 78 +-- .../DeliveryManagerPriorityTests.scala | 9 +- .../DeliveryManagerRequestModifiesSpec.scala | 587 ++++++++---------- 4 files changed, 313 insertions(+), 363 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 841b48cf5e..52adcc35b8 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -40,7 +40,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifier += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifier.contains(toKey(modId))) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 03d54c59cb..0f62964c0d 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -1,11 +1,12 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress + import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} import encry.local.miner.Miner.{DisableMining, StartMining} import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager +import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} @@ -14,44 +15,43 @@ import encry.view.history.History import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.mutable import scala.collection.mutable.WrappedArray -//object DMUtils extends InstanceFactory { -// -// def initialiseDeliveryManager(isBlockChainSynced: Boolean, -// isMining: Boolean, -// settings: EncryAppSettings) -// (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { -// val history: History = generateDummyHistory(settings) -// val deliveryManager: TestActorRef[DeliveryManager] = -// TestActorRef[DeliveryManager](DeliveryManager -// .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) -// deliveryManager ! UpdatedHistory(history) -// if (isMining) deliveryManager ! StartMining -// else deliveryManager ! DisableMining -// if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced -// (deliveryManager, history) -// } -// -// def generateBlocks(qty: Int, history: History): (History, List[Block]) = -// (0 until qty).foldLeft(history, List.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// val a = prevHistory.reportModifierIsValid(block) -// (a, blocks :+ block) -// } -// -// def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) -// -// def createPeer(port: Int, -// host: String, -// settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { -// val address = new InetSocketAddress(host, port) -// val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, -// Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) -// (address, peer) -// } -//} \ No newline at end of file +object DMUtils extends InstanceFactory { + + def initialiseDeliveryManager(isBlockChainSynced: Boolean, + isMining: Boolean, + settings: EncryAppSettings) + (implicit actorSystem: ActorSystem): (TestActorRef[DM], History) = { + val history: History = generateDummyHistory(settings) + val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network)) + deliveryManager ! UpdatedHistory(history) + if (isMining) deliveryManager ! StartMining + else deliveryManager ! DisableMining + if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced + (deliveryManager, history) + } + + def generateBlocks(qty: Int, history: History): (History, List[Block]) = + (0 until qty).foldLeft(history, List.empty[Block]) { + case ((prevHistory, blocks), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory.append(block.header) + prevHistory.append(block.payload) + val a = prevHistory.reportModifierIsValid(block) + (a, blocks :+ block) + } + + def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + + def createPeer(port: Int, + host: String, + settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { + val address = new InetSocketAddress(host, port) + val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, + Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) + (address, peer) + } +} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala index fb358c98b8..a2a0752e80 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala @@ -2,12 +2,14 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -//import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} +import encry.network.DM +import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit} import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.modifiers.InstanceFactory +import encry.network.DM.RequestStatus import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer //import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal @@ -32,7 +34,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecL // // override def afterAll: Unit = TestKit.shutdownActorSystem(system) // -// def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, +// def initialiseState: (TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, // ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, // List[Block], List[ModifierId]) = { // val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) @@ -66,8 +68,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecL // val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState // val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = // Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestStatus(cp1, Header.modifierTypeId, headersIds) // val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic // assert(result.contains(cp1.socketAddress)) // assert(result(cp1.socketAddress) == BadNode) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 8a12695db0..f0e6f1215e 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,319 +1,268 @@ -//package encry.network.DeliveryManagerTests -// -//import java.net.InetSocketAddress -// -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestKit, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.settings.TestNetSettings -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.Messages.MessageToNetwork.RequestFromLocal -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -//import org.encryfoundation.common.network.SyncInfo -//import org.encryfoundation.common.utils.Algos -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -// -//import scala.collection.mutable.WrappedArray -// -//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { -// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) -// } -// -// "RequestModifies" should { -// "handle uniq modifiers from RequestFromLocal message correctly" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "not handle repeating modifiers from RequestFromLocal message" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery Manager should handle received modifier which were requested correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery manager should not handle repeating modifiers" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "handle priority request for payload correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) -// headersIds.foreach(id => -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => -// block.id.sameElements(id)).get.payload.id)) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .size == blocks.size) -// deliveryManager.stop() -// } -// "choose correct peer in priority request" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map( -// address1 -> (cp1, Older, InitialPriority), -// address2 -> (cp2, Older, InitialPriority), -// address3 -> (cp3, Older, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) -// -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) -// -// handler1.expectMsgAnyOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), -// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), -// SyncInfoNetworkMessage(SyncInfo(List())) -// ) -// -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers while block chain is not synced from Younger nodes" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection = -// Map( -// address2 -> (cp2, Younger, InitialPriority), -// address3 -> (cp3, Fork, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// handler2.expectNoMsg() -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers from peer which is not contained in status tracker" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address2 -> (cp2, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask transactions while block chain is not synced" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not ask transaction while node is not mining" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not re-ask modifiers which already have been received" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) -// ) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) -// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) -// deliveryManager.stop() -// } -// } -//} \ No newline at end of file +package encry.network.DeliveryManagerTests + +import java.net.InetSocketAddress + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestKit, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +import encry.modifiers.InstanceFactory +import encry.network.DM.RequestSent +import encry.network.{DM, DeliveryManager} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.settings.TestNetSettings +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.collection.mutable.WrappedArray + +class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = TestKit.shutdownActorSystem(system) + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { + val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + } + + "RequestModifies" should { + "handle uniq modifiers from RequestFromLocal message correctly" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "not handle repeating modifiers from RequestFromLocal message" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery Manager should handle received modifier which were requested correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifier.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery manager should not handle repeating modifiers" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + assert(deliveryManager.underlyingActor.receivedModifier.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "handle priority request for payload correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1.socketAddress) + headersIds.foreach(id => + deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, blocks.find(block => + block.id.sameElements(id)).get.payload.id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == blocks.size) + deliveryManager.stop() + } + "choose correct peer in priority request" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map( + address1 -> (cp1, Older, InitialPriority), + address2 -> (cp2, Older, InitialPriority), + address3 -> (cp3, Older, InitialPriority) + ) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) + + deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) + + handler1.expectMsgAnyOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), + RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), + SyncInfoNetworkMessage(SyncInfo(List())) + ) + + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers from peer which is not contained in status tracker" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address2 -> (cp2, Older, InitialPriority)) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) + + handler1.expectNoMsg() + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask transactions while block chain is not synced" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + txs.foreach(tx => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, tx.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not ask transaction while node is not mining" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + txs.foreach(tx => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, tx.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not re-ask modifiers which already have been received" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1.socketAddress) + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifier.size == 1) + assert(deliveryManager.underlyingActor.receivedModifier.contains(toKey(header.id))) + deliveryManager.stop() + } + } +} \ No newline at end of file From 35e32b5a7a5ae8fd3d59806d968d4a46e8eea45b Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 16:21:03 +0300 Subject: [PATCH 094/119] more dm tests --- ...DeliveryManagerReRequestModifiesSpec.scala | 285 ++++++++---------- .../DeliveryManagerRequestModifiesSpec.scala | 12 +- 2 files changed, 131 insertions(+), 166 deletions(-) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index 106005b9ff..e29d0daafd 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -1,55 +1,59 @@ -//package encry.network.DeliveryManagerTests -// -//import java.net.InetSocketAddress -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.Older -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -////import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.NodeViewSynchronizer.ReceivableMessages._ -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.settings.TestNetSettings -//import encry.view.history.History -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import scala.concurrent.duration._ -//import scala.collection.mutable.WrappedArray -// -//class DeliveryManagerReRequestModifiesSpec extends WordSpecLike -// with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = system.terminate() -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { -// val (deliveryManager, history) = -// initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) -// } -// -// "ReRequestModifies" should { -// "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + +package encry.network.DeliveryManagerTests + +import java.net.InetSocketAddress + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.Older +import encry.modifiers.InstanceFactory +import encry.network.DM.RequestSent +import encry.network.{DM, DeliveryManager} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.settings.TestNetSettings +import encry.view.history.History +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} + +import scala.concurrent.duration._ +import scala.collection.mutable.WrappedArray + +class DeliveryManagerReRequestModifiesSpec extends WordSpecLike + with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = system.terminate() + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { + val (deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) + } + + "ReRequestModifies" should { + "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + + //todo: move to message builder tests // "expectedModifiers collection after all attempts will expire" in { // val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() // @@ -66,7 +70,7 @@ // // val header: ModifierId = headersIds.head // -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) // handler1.expectMsgAllOf( // testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), // RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), @@ -76,112 +80,73 @@ // //this thread sleep is using for expecting modifier removal // Thread.sleep(6000) // -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// deliveryManager.stop() -// } -// "not re-ask unnecessary modifiers" in { -// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: ModifierId = headersIds.head -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) -// -// //await one re-ask -// handler1.expectMsgAllOf( -// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) -// ) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, -// Map(header -> headerBytes)), cp1) -// deliveryManager.stop() -// } -// "not re-ask modifiers which were applied to the history" in { -// val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// -// handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, -// Map(headerIds.head -> headerBytes)), cp1) -// -// history.append(blocks.head.header) -// val uHistory: History = history.reportModifierIsValid(blocks.head.header) -// -// deliveryManager ! UpdatedHistory(uHistory) -// -// deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// deliveryManager.stop() -// } -// "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { -// val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// //this thread sleep is using for expecting modifier removal -// Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) -// deliveryManager.stop() -// } -// "not re-ask transactions" in { -// val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) -// ) -// handler1.expectNoMsg(10.seconds) -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) // deliveryManager.stop() // } -// } -//} \ No newline at end of file + "not re-ask unnecessary modifiers" in { + val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: ModifierId = headersIds.head + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) + + //await one re-ask + handler1.expectMsgAllOf( + testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) + ) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, + Map(header -> headerBytes)), cp1.socketAddress) + deliveryManager.stop() + } + "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { + val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, headerIds.head) + //this thread sleep is using for expecting modifier removal + Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 2000) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } + "not re-ask transactions" in { + val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) + + transactions.foreach(txId => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, txId)) + + handler1.expectMsgAllOf( + RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) + ) + handler1.expectNoMsg(10.seconds) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } + } +} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index f0e6f1215e..3e2d241362 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -81,8 +81,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager ! DataFromPeer(ModifiersNetworkMessage( Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifier.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) deliveryManager.stop() } "Delivery manager should not handle repeating modifiers" in { @@ -96,8 +96,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) deliveryManager ! DataFromPeer(ModifiersNetworkMessage( Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) - assert(deliveryManager.underlyingActor.receivedModifier.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) deliveryManager.stop() } "handle priority request for payload correctly" in { @@ -260,8 +260,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifier.size == 1) - assert(deliveryManager.underlyingActor.receivedModifier.contains(toKey(header.id))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) + assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) deliveryManager.stop() } } From e08d76b3914c94adfe839d5b6d86c35ede1aad80 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 17 Mar 2020 11:52:39 +0300 Subject: [PATCH 095/119] fix some bugs in dm and pk --- src/main/scala/encry/network/DM.scala | 18 +- src/main/scala/encry/network/PK.scala | 1 + .../scala/encry/network/BlackListTests.scala | 193 +++++++++--------- .../history/ModifiersValidationTest.scala | 90 ++++---- 4 files changed, 151 insertions(+), 151 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 52adcc35b8..dbd2a2fc1f 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -26,7 +26,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte var expectedModifiers: Set[ModifierIdAsKey] = Set.empty - var receivedModifier: Set[ModifierIdAsKey] = Set.empty + var receivedModifiers: Set[ModifierIdAsKey] = Set.empty override def preStart(): Unit = context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) @@ -37,16 +37,16 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging if (expectedModifiers.contains(toKey(id))) { context.parent ! ModifierFromNetwork(source, data._1, id, bytes) expectedModifiers -= toKey(id) - receivedModifier += toKey(id) + receivedModifiers += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifier.contains(toKey(modId))) => + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case RequestSent(_, _, _) => //do nothing - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts < networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) logger.info(s"Re-request modifier ${Algos.encode(modId)}") context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! @@ -59,16 +59,16 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) - receivedModifier += toKey(modId) + receivedModifiers += toKey(modId) context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") - case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) - case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) + case SemanticallySuccessfulModifier(mod) => receivedModifiers -= toKey(mod.id) + case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) case IsRequested(modIds) => //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") sender ! RequestStatus( - requested = modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), - notRequested = modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + requested = modIds.filter(id => receivedModifiers.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + notRequested = modIds.filter(id => !receivedModifiers.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) ) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 8b175cfcd2..ed3a9cdcf6 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -57,6 +57,7 @@ class PK(networkSettings: NetworkSettings, blackList.getAll.toList ) } + context.system.scheduler.schedule(600.millis, blacklistSettings.cleanupTime){blackList = blackList.cleanupBlackList} } override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index d71faf7d77..ada46f14a7 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -14,100 +14,99 @@ import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -//class BlackListTests extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem() -// -// override def afterAll(): Unit = system.terminate() -// -// val knowPeersSettings = testNetSettings.copy( -// network = settings.network.copy( -// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), -// connectOnlyWithKnownPeers = Some(true) -// ), -// blackList = settings.blackList.copy( -// banTime = 2 seconds, -// cleanupTime = 3 seconds -// )) -// -// /* -// Unit tests -// */ -// "Black list" should { -// "temporary ban requested peer correctly" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) -// newBL.contains(peer) shouldBe true -// } -// "clean black list from peers with expired ban time which were banned by temporary ban" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) -// Thread.sleep(2000) -// val newBL1 = newBL.cleanupBlackList -// newBL1.contains(peer) shouldBe false -// } -// "don't remove peer from black list before ban time expired" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SentInvForPayload, peer) -// val newBL1 = newBL.cleanupBlackList -// newBL1.contains(peer) shouldBe true -// } -// } -// -// /* -// Akka tests -// */ -// "Peers keeper" should { -// "handle ban peer message correctly" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// peersKeeper ! BanPeer(connectedPeer, SpamSender) -// peerHandler.expectMsg(CloseConnection) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true -// } -// "cleanup black list by scheduler correctly" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) -// Thread.sleep(6000) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false -// } -// "don't remove peer from black list before ban time expired" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// Thread.sleep(4000) -// peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) -// Thread.sleep(2000) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true -// } -// } -//} \ No newline at end of file +class BlackListTests extends WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem() + + override def afterAll(): Unit = system.terminate() + + val knowPeersSettings = testNetSettings.copy( + network = settings.network.copy( + knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), + connectOnlyWithKnownPeers = Some(true) + ), + blackList = settings.blackList.copy( + banTime = 2 seconds, + cleanupTime = 3 seconds + )) + + /* + Unit tests + */ + "Black list" should { + "temporary ban requested peer correctly" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) + newBL.contains(peer) shouldBe true + } + "clean black list from peers with expired ban time which were banned by temporary ban" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) + Thread.sleep(2000) + val newBL1 = newBL.cleanupBlackList + newBL1.contains(peer) shouldBe false + } + "don't remove peer from black list before ban time expired" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SentInvForPayload, peer) + val newBL1 = newBL.cleanupBlackList + newBL1.contains(peer) shouldBe true + } + } + + /* + Akka tests + */ + "Peers keeper" should { + "handle ban peer message correctly" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList)) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + peersKeeper ! BanPeer(connectedPeer.socketAddress, SpamSender) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true + } + "cleanup black list by scheduler correctly" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList.copy(banTime = 1 millisecond, cleanupTime = 1 millisecond))) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + peersKeeper ! BanPeer(connectedPeer.socketAddress, SentPeersMessageWithoutRequest) + Thread.sleep(6000) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false + } + "don't remove peer from black list before ban time expired" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList)) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + Thread.sleep(4000) + peersKeeper ! BanPeer(connectedPeer.socketAddress, CorruptedSerializedBytes) + Thread.sleep(2000) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true + } + } +} \ No newline at end of file diff --git a/src/test/scala/encry/view/history/ModifiersValidationTest.scala b/src/test/scala/encry/view/history/ModifiersValidationTest.scala index 722258dc54..e879864ea8 100644 --- a/src/test/scala/encry/view/history/ModifiersValidationTest.scala +++ b/src/test/scala/encry/view/history/ModifiersValidationTest.scala @@ -1,45 +1,45 @@ -//package encry.view.history -// -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -//import encry.settings.{EncryAppSettings, TestNetSettings} -//import org.encryfoundation.common.modifiers.history.Block -//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} -// -//class ModifiersValidationTest extends WordSpecLike -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// "Modifiers validator" should { -// "validate genesis block" in { -// val newHistory: History = generateDummyHistory(testNetSettings) -// val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) -// newHistory.testApplicable(genesisBlock.header).isRight shouldBe true -// newHistory.append(genesisBlock.header) -// val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) -// updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true -// } -// "reject incorrect modifiers" in { -// val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 -// val newHistory: History = generateDummyHistory(testNetSettings) -// blocks.take(1).foldLeft(newHistory) { case (history, block) => -// history.testApplicable(block.header).isRight shouldBe true -// history.append(block.header) -// history.reportModifierIsValid(block.header) -// history.testApplicable(block.payload).isRight shouldBe true -// history.append(block.payload) -// history.reportModifierIsValid(block) -// } -// blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => -// history.testApplicable(block.header).isRight shouldBe false -// history.append(block.header) -// history.reportModifierIsValid(block.header) -// history.testApplicable(block.payload).isRight shouldBe true -// history.append(block.payload) -// history.reportModifierIsValid(block) -// } -// } -// } -//} \ No newline at end of file +package encry.view.history + +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +import encry.settings.{EncryAppSettings, TestNetSettings} +import org.encryfoundation.common.modifiers.history.Block +import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} + +class ModifiersValidationTest extends WordSpecLike + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + "Modifiers validator" should { + "validate genesis block" in { + val newHistory: History = generateDummyHistory(testNetSettings) + val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) + newHistory.testApplicable(genesisBlock.header).isRight shouldBe true + newHistory.append(genesisBlock.header) + val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) + updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true + } + "reject incorrect modifiers" in { + val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 + val newHistory: History = generateDummyHistory(testNetSettings) + blocks.take(1).foldLeft(newHistory) { case (history, block) => + history.testApplicable(block.header).isRight shouldBe true + history.append(block.header) + history.reportModifierIsValid(block.header) + history.testApplicable(block.payload).isRight shouldBe true + history.append(block.payload) + history.reportModifierIsValid(block) + } + blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => + history.testApplicable(block.header).isRight shouldBe false + history.append(block.header) + history.reportModifierIsValid(block.header) + history.testApplicable(block.payload).isRight shouldBe true + history.append(block.payload) + history.reportModifierIsValid(block) + } + } + } +} \ No newline at end of file From 8f216f0c3c10281e28c437e7f8499352ad4629e9 Mon Sep 17 00:00:00 2001 From: lior Date: Tue, 17 Mar 2020 13:54:19 +0300 Subject: [PATCH 096/119] test added --- .../encry/view/mempool/MemoryPoolTests.scala | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index 04e5b67301..c805e925d3 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -1,16 +1,25 @@ package encry.view.mempool +import java.net.InetSocketAddress + +import scala.concurrent.duration._ import akka.actor.ActorSystem -import akka.testkit.{ TestActorRef, TestProbe } +import akka.testkit.{TestActorRef, TestProbe} import com.typesafe.scalalogging.StrictLogging import encry.modifiers.InstanceFactory -import encry.mpg.MemoryPool._ -import encry.mpg.{ MemoryPool, MemoryPoolStorage } +import encry.mpg.MemoryPool.{RolledBackTransactions, TransactionProcessing, UpdateMempoolReader} +import encry.mpg.{IntermediaryMempool, MemoryPool, MemoryPoolProcessor, MemoryPoolReader, MemoryPoolStorage, TransactionsValidator} +import encry.network.BlackList.BanReason.SemanticallyInvalidPersistentModifier +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer import encry.settings.TestNetSettings import encry.utils.NetworkTimeProvider -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } - -import scala.concurrent.duration._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} class MemoryPoolTests extends WordSpecLike @@ -58,5 +67,27 @@ class MemoryPoolTests txs.map(_.encodedId).forall(transactions.map(_.encodedId).contains) shouldBe true transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } + "chainSynced is true on FullBlockChainIsSynced" in { + val mP = TestActorRef[MemoryPoolProcessor](MemoryPoolProcessor.props(settings, timeProvider)) + mP ! FullBlockChainIsSynced + mP.underlyingActor.chainSynced shouldBe true + } + "storage changes on RolledBackTransactions" in { + val fakeActor = TestProbe() + val storage = MemoryPoolStorage.empty(testNetSettings, timeProvider) + val memPool = TestActorRef[MemoryPool](MemoryPool.props(settings, timeProvider, Some(fakeActor.ref), fakeActor.ref)) + val txs = (0 until 10).map(k => coinbaseAt(k)) + memPool ! RolledBackTransactions(txs) + assert(memPool.underlyingActor.memoryPool != storage) + memPool.underlyingActor.memoryPool.size shouldBe txs.length + } + "TransactionProcessing" in { + val mP = TestActorRef[MemoryPoolProcessor](MemoryPoolProcessor.props(settings, timeProvider)) + mP ! TransactionProcessing(true) + mP.underlyingActor.canProcessTransactions shouldBe true + mP ! TransactionProcessing(false) + mP.underlyingActor.canProcessTransactions shouldBe false + } + } } From cb3a283110801306d5e0828dbe9fad387c4ba2d1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:29:29 +0300 Subject: [PATCH 097/119] sync info message tests added --- .../scala/encry/network/DeliveryManager.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 4 +- src/main/scala/encry/network/PK.scala | 2 +- .../scala/encry/network/PeersKeeper.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 10 +- .../scala/encry/view/history/HistoryApi.scala | 2 +- .../history/HistoryPayloadsProcessor.scala | 28 +- src/test/scala/encry/nvg/NVHTests.scala | 5 + .../nvg/NetworkMessagesProcessorTests.scala | 284 ++++++++++++++++++ src/test/scala/encry/nvg/PipelinesTests.scala | 5 + src/test/scala/encry/nvg/Utils.scala | 24 ++ src/test/scala/encry/nvg/ValidatorTests.scala | 5 + 13 files changed, 355 insertions(+), 20 deletions(-) create mode 100644 src/test/scala/encry/nvg/NVHTests.scala create mode 100644 src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala create mode 100644 src/test/scala/encry/nvg/PipelinesTests.scala create mode 100644 src/test/scala/encry/nvg/Utils.scala create mode 100644 src/test/scala/encry/nvg/ValidatorTests.scala diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 47c475f688..5e76c06349 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -538,7 +538,7 @@ object DeliveryManager { case StartTransactionsValidation => 2 - case OtherNodeSyncingStatus(_, _, _) => 1 + case OtherNodeSyncingStatus(_, _) => 1 case ConnectionStopped(_) => 1 diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 45fd0cde77..4054bf3772 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -220,9 +220,7 @@ object NodeViewSynchronizer { case object SendLocalSyncInfo - final case class OtherNodeSyncingStatus(remote: InetSocketAddress, - status: encry.consensus.HistoryConsensus.HistoryComparisonResult, - extension: Option[Seq[(ModifierTypeId, ModifierId)]]) + final case class OtherNodeSyncingStatus(remote: InetSocketAddress, status: HistoryComparisonResult) sealed trait CLIPeer diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index ed3a9cdcf6..3c2efb1c5a 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -85,7 +85,7 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } - case OtherNodeSyncingStatus(remote, comparison, _) => + case OtherNodeSyncingStatus(remote, comparison) => connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index dc92e76c20..72dc6e85bb 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -352,7 +352,7 @@ object PeersKeeper { class PeersKeeperPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case OtherNodeSyncingStatus(_, _, _) => 0 + case OtherNodeSyncingStatus(_, _) => 0 case AccumulatedPeersStatistic(_) => 1 case BanPeer(_, _) => 1 case SendLocalSyncInfo => 1 diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 4bcf66def2..cd74130cbd 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -98,7 +98,7 @@ class IntermediaryNVH( case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 13761962ca..58832e39eb 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -65,15 +65,13 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - val ext: Seq[ModifierId] = - historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + logger.info(s"Got sync info network message. This message includes ${syncInfo.lastHeaderIds.size} ids.") val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( - s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + - s"Comparison result is $comparison. Sending extension of length ${ext.length}." + s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n " + + s"Comparison result is $comparison. \n " ) - if (!(ext.nonEmpty || comparison != Younger)) logger.info("Extension is empty while comparison is younger") - context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) + context.parent ! OtherNodeSyncingStatus(remote, comparison) case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => logger.info( diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 4b8c6c7e92..91f5090cb8 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -216,7 +216,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore }.toList }.getOrElse(List.empty)) - def compare(si: SyncInfo): HistoryComparisonResult = lastSyncInfo.lastHeaderIds.lastOption match { + def compare(si: SyncInfo): HistoryComparisonResult = syncInfo.lastHeaderIds.lastOption match { //Our best header is the same as other history best header case Some(id) if si.lastHeaderIds.lastOption.exists(_ sameElements id) => Equal //Our best header is in other history best chain, but not at the last position diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index 071e547cf2..9732bd6af6 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -5,8 +5,9 @@ import cats.syntax.option._ import encry.consensus.HistoryConsensus.ProgressInfo import encry.modifiers.history.HeaderChain import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} trait HistoryPayloadsProcessor extends HistoryApi { @@ -27,7 +28,8 @@ trait HistoryPayloadsProcessor extends HistoryApi { processValidFirstBlock(blockToProcess, header, bestFullChain) case Some(header) if isBestBlockDefined && isBetterChain(header.id) => processBetterChain(blockToProcess, header, Seq.empty, settings.node.blocksToKeep) - case Some(_) => + case Some(header) => + logger.info(s"\n\nnonBestBlock. id: ${blockToProcess.header.encodedId}. cause: ${isBestBlockDefined} or ${ isBetterChain(header.id)}\n\n") nonBestBlock(blockToProcess) case None => logger.debug(s"Best full chain is empty. Returning empty progress info") @@ -54,7 +56,10 @@ trait HistoryPayloadsProcessor extends HistoryApi { val toApply: Seq[Block] = newChain.tail.headers .flatMap(h => if (h == fullBlock.header) fullBlock.some else getBlockByHeader(h)) toApply.foreach(addBlockToCacheIfNecessary) - if (toApply.lengthCompare(newChain.length - 1) != 0) nonBestBlock(fullBlock) + if (toApply.lengthCompare(newChain.length - 1) != 0) { + logger.info(s"To apply. processBetterChain. nonBestBlock.") + nonBestBlock(fullBlock) + } else { //application of this block leads to full chain with higher score logger.info(s"Appending ${fullBlock.encodedId}|${fullBlock.header.height} as a better chain") @@ -104,8 +109,19 @@ trait HistoryPayloadsProcessor extends HistoryApi { prevBestScore <- scoreOf(bestFullBlockId) score <- scoreOf(id) bestBlockHeight = getBestBlockHeight - } yield (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)) - .getOrElse(false) + } yield { + logger.info(s"isBetterChain. id: ${Algos.encode(id)}. \n " + + s"bestBlockHeight: $bestBlockHeight.\n " + + s"heightOfThisHeader $heightOfThisHeader.\n " + + s"score: $score.\n " + + s"prevBestScore: $prevBestScore.\n " + + s"res is: ${(bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)}") + (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore) + }) + .getOrElse { + logger.info(s"isBetterChain. id: ${Algos.encode(id)}. getOrElse. false.") + false + } private def calculateBestFullChain(block: Block): Seq[Block] = { val continuations: Seq[Seq[Header]] = continuationHeaderChains(block.header, h => isBlockDefined(h)).map(_.tail) diff --git a/src/test/scala/encry/nvg/NVHTests.scala b/src/test/scala/encry/nvg/NVHTests.scala new file mode 100644 index 0000000000..3f7c4728ee --- /dev/null +++ b/src/test/scala/encry/nvg/NVHTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class NVHTests { + +} diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala new file mode 100644 index 0000000000..896b4e857d --- /dev/null +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -0,0 +1,284 @@ +package encry.nvg + +import java.net.InetSocketAddress + +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } +import cats.syntax.eq._ +import encry.consensus.HistoryConsensus.{ Equal, Older, Unknown, Younger } +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager.CheckPayloadsToDownload +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.Utils.instances._ +import encry.settings.EncryAppSettings +import encry.view.history.{ History, HistoryReader } +import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, SyncInfoNetworkMessage } +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } + +class NetworkMessagesProcessorTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest { + + "Network messages processor" should { + "process sync info message correctly" should { + "determine older extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (historyMain: History, historyOlder: History) = + NetworkMessagesProcessorTests.formYoungerActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(historyMain) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine younger extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (historyMain: History, historyYounger: History) = + NetworkMessagesProcessorTests.formOlderActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(historyMain) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine equals extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history1: History, history2: History) = + NetworkMessagesProcessorTests.fromEqualActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(history1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine unknown extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (h1: History, h2: History) = NetworkMessagesProcessorTests.fromEUnknownActorState + + val historyReader: HistoryReader = HistoryReader(h1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + } + "process inv message correctly" in {} + "process request for modifier correctly" in {} + "have correct logic with check payloads to download" in {} + "have correct logic with semantically successful modifier" in {} + } +} + +object NetworkMessagesProcessorTests extends InstanceFactory { + + def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NetworkMessagesProcessor] = { + val networkProcessor: TestActorRef[NetworkMessagesProcessor] = + TestActorRef[NetworkMessagesProcessor](NetworkMessagesProcessor.props(settings)) + networkProcessor + } + + def formDataFromPeerMessage(innerMessage: NetworkMessage, host: String, port: Int)( + implicit AS: ActorSystem + ): (DataFromPeer, InetSocketAddress) = { + val address = new InetSocketAddress(host, port) + DataFromPeer(innerMessage, address) -> address + } + + def formYoungerActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { + val (hMain, hOlder, blocks) = (0 until blocksQty).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings), + List.empty[Block] + ) { + case ((historyMain, historyOlder, blocks: List[Block]), _) => + val block: Block = generateNextBlock(historyMain) + val hMain: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hOlder = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hMain, hOlder, block +: blocks) + } + val (hOlderUpdated, _) = (0 until olderBlocksQty).foldLeft(hOlder, List.empty[Block]) { + case ((historyOlder, blocks: List[Block]), _) => + val block: Block = generateNextBlock(historyOlder) + val hOlder: History = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hOlder, block +: blocks) + } + (hMain, hOlderUpdated) + } + + def formOlderActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { + val (historyYounger, historyOlder) = formYoungerActorState(blocksQty, olderBlocksQty) + (historyOlder, historyYounger) + } + + def fromEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = + (0 until blocksQty).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings) + ) { + case ((historyMain, historyOlder), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hEq2 = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hEq1, hEq2) + } + + def fromEUnknownActorState: (History, History) = { + val history1 = (0 until 100).foldLeft( + generateDummyHistory(settings) + ) { + case (h1, _) => + val block1: Block = generateNextBlock(h1) + val hEq1: History = h1 + .append(block1.header) + .right + .get + ._1 + .append(block1.payload) + .right + .get + ._1 + .reportModifierIsValid(block1) + hEq1 + } + val history2 = (0 until 2).foldLeft( + generateDummyHistory(settings) + ) { + case (h2, _) => + val block1: Block = generateNextBlock(h2) + val hEq2: History = h2 + .append(block1.header) + .right + .get + ._1 + .append(block1.payload) + .right + .get + ._1 + .reportModifierIsValid(block1) + hEq2 + } + history1 -> history2 + } + +} diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala new file mode 100644 index 0000000000..05d227e223 --- /dev/null +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class PipelinesTests { + +} diff --git a/src/test/scala/encry/nvg/Utils.scala b/src/test/scala/encry/nvg/Utils.scala new file mode 100644 index 0000000000..54c6ae0d34 --- /dev/null +++ b/src/test/scala/encry/nvg/Utils.scala @@ -0,0 +1,24 @@ +package encry.nvg + +import cats.Eq +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import cats.instances.option._ +import cats.instances.int._ +import cats.instances.byte._ +import cats.syntax.eq._ + +object Utils { + object instances { + implicit object ModifierIdEq extends Eq[ModifierId] { + override def eqv(x: ModifierId, y: ModifierId): Boolean = + x.sameElements(y) + } + + implicit object OtherNodeSyncStatusEq extends Eq[OtherNodeSyncingStatus] { + override def eqv(x: OtherNodeSyncingStatus, y: OtherNodeSyncingStatus): Boolean = + x.status == y.status && x.remote == y.remote + } + } + +} diff --git a/src/test/scala/encry/nvg/ValidatorTests.scala b/src/test/scala/encry/nvg/ValidatorTests.scala new file mode 100644 index 0000000000..72bcac2eed --- /dev/null +++ b/src/test/scala/encry/nvg/ValidatorTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class ValidatorTests { + +} From 204db4e048745d9e527e5c97099f38825ed6ce73 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:30:20 +0300 Subject: [PATCH 098/119] fixed modifiers propagation --- src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 896b4e857d..09544360f9 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -1,7 +1,6 @@ package encry.nvg import java.net.InetSocketAddress - import akka.actor.{ ActorRef, ActorSystem } import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ From 2d084cffb361589ae5874c8419636d52636736a8 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:43:22 +0300 Subject: [PATCH 099/119] added test for fork in sync info --- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +-- .../nvg/NetworkMessagesProcessorTests.scala | 113 ++++++++++++++++-- 2 files changed, 111 insertions(+), 20 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 58832e39eb..e3a819d6ea 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -62,17 +62,17 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) case _ => } + + case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => + val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) + logger.info( + s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + + s"Comparison result is $comparison. \n " + ) + context.parent ! OtherNodeSyncingStatus(remote, comparison) + case DataFromPeer(message, remote) => message match { - case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - logger.info(s"Got sync info network message. This message includes ${syncInfo.lastHeaderIds.size} ids.") - val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) - logger.info( - s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n " + - s"Comparison result is $comparison. \n " - ) - context.parent ! OtherNodeSyncingStatus(remote, comparison) - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => logger.info( s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 09544360f9..cbdb0f3c4d 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -1,10 +1,11 @@ package encry.nvg import java.net.InetSocketAddress -import akka.actor.{ ActorRef, ActorSystem } -import akka.testkit.{ TestActorRef, TestKit, TestProbe } + +import akka.actor.{ActorRef, ActorSystem} +import akka.testkit.{TestActorRef, TestKit, TestProbe} import cats.syntax.eq._ -import encry.consensus.HistoryConsensus.{ Equal, Older, Unknown, Younger } +import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.NetworkController.ReceivableMessages.DataFromPeer @@ -12,10 +13,10 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingSta import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings -import encry.view.history.{ History, HistoryReader } +import encry.view.history.{History, HistoryReader} import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, SyncInfoNetworkMessage } -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +import org.encryfoundation.common.network.BasicMessagesRepo.{NetworkMessage, SyncInfoNetworkMessage} +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -87,7 +88,7 @@ class NetworkMessagesProcessorTests val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) val (history1: History, history2: History) = - NetworkMessagesProcessorTests.fromEqualActorState(10, 10) + NetworkMessagesProcessorTests.formEqualActorState(10, 10) val historyReader: HistoryReader = HistoryReader(history1) @@ -113,7 +114,7 @@ class NetworkMessagesProcessorTests val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) - val (h1: History, h2: History) = NetworkMessagesProcessorTests.fromEUnknownActorState + val (h1: History, h2: History) = NetworkMessagesProcessorTests.formUnknownActorState val historyReader: HistoryReader = HistoryReader(h1) @@ -128,6 +129,33 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine fork extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (h1: History, h2: History) = + NetworkMessagesProcessorTests.formForkActorState(10, 20, 5) + + val historyReader: HistoryReader = HistoryReader(h1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) + + networkProcessor ! dataFromPeerMsg + parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => msg eqv expectedResult case CheckPayloadsToDownload => @@ -211,7 +239,7 @@ object NetworkMessagesProcessorTests extends InstanceFactory { (historyOlder, historyYounger) } - def fromEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = + def formEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = (0 until blocksQty).foldLeft( generateDummyHistory(settings), generateDummyHistory(settings) @@ -242,8 +270,8 @@ object NetworkMessagesProcessorTests extends InstanceFactory { (hEq1, hEq2) } - def fromEUnknownActorState: (History, History) = { - val history1 = (0 until 100).foldLeft( + def formUnknownActorState: (History, History) = { + val history1 = (0 until 10).foldLeft( generateDummyHistory(settings) ) { case (h1, _) => @@ -280,4 +308,67 @@ object NetworkMessagesProcessorTests extends InstanceFactory { history1 -> history2 } + def formForkActorState(forkOn: Int, forkSize: Int, mainChainSize: Int): (History, History) = { + val (h1, h2) = (0 until forkOn).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings) + ) { + case ((historyMain, historyOlder), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hEq2 = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hEq1, hEq2) + } + val fork = (0 until forkSize).foldLeft(h1) { + case (historyMain, _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 + } + val mch = (0 until mainChainSize).foldLeft(h2) { + case (historyMain, _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 + } + mch -> fork + } + } From 61ab55b6c4e624bdb17f85de49e897636d459fbe Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 17:17:30 +0300 Subject: [PATCH 100/119] added tests for processing inv message on nvh messages processor --- .../encry/nvg/NetworkMessagesProcessor.scala | 47 ++++--- .../nvg/NetworkMessagesProcessorTests.scala | 131 ++++++++++++++++-- src/test/scala/encry/nvg/Utils.scala | 13 +- 3 files changed, 156 insertions(+), 35 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index e3a819d6ea..672154b22e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -15,6 +15,7 @@ import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage, @@ -22,7 +23,8 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + import scala.concurrent.duration._ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { @@ -67,34 +69,33 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + - s"Comparison result is $comparison. \n " + s"Comparison result is $comparison.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) - case DataFromPeer(message, remote) => - message match { - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => - logger.info( - s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + - s"But full chain is not synced. Ignore them." - ) - - case InvNetworkMessage(invData) => - val startTime: Long = System.currentTimeMillis() - val ids: Seq[ModifierId] = invData._2.filterNot( - (mid: ModifierId) => - historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) - ) + case DataFromPeer(InvNetworkMessage(invData: InvData), remote) => + if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) + logger.info(s"Got inv message from $remote with ${invData._2.size} ids but full chain is not synced.") + else { + val isHeader: Boolean = invData._1 == Header.modifierTypeId + val isPayloadAvailable: Boolean = + historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId + val isRequestAvailable: Boolean = isHeader || isPayloadAvailable + if (isRequestAvailable) { + val ids: Seq[ModifierId] = invData._2.filterNot { mid: ModifierId => + historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) + } + logger.info(s"Sending request from local to $remote with ${ids.size} ids of type ${invData._1}.") + if (ids.nonEmpty) context.parent ! RequestFromLocal(remote.some, invData._1, ids.toList) + } else logger.info( - s"Got inv message from $remote. Type of nested ids is: ${invData._1}. " + - s"Ids for request are: ${ids.map(Algos.encode).mkString(",")}." + s"Got inv message from $remote but response is unavailable cause:" + + s" is header - $isHeader || is payload available - $isPayloadAvailable." ) - if (ids.nonEmpty && - (invData._1 == Header.modifierTypeId || - (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) - sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) - logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") + } + case DataFromPeer(message, remote) => + message match { case RequestModifiersNetworkMessage((typeId, requestedIds)) => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap( diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index cbdb0f3c4d..48f6f37492 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -5,18 +5,26 @@ import java.net.InetSocketAddress import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestActorRef, TestKit, TestProbe} import cats.syntax.eq._ +import cats.syntax.option._ import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings import encry.view.history.{History, HistoryReader} -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.BasicMessagesRepo.{NetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.utils.TaggedTypes +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import scorex.utils.Random +import supertagged.@@ + +import scala.collection.immutable class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -26,6 +34,8 @@ class NetworkMessagesProcessorTests with InstanceFactory with OneInstancePerTest { + override def afterAll(): Unit = system.terminate() + "Network messages processor" should { "process sync info message correctly" should { "determine older extension" in { @@ -50,7 +60,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -77,7 +87,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -104,7 +114,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -130,7 +140,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -157,13 +167,102 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } } } - "process inv message correctly" in {} + "process inv message correctly" should { + "not process inv for payload while full chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) + + parentActor.expectNoMsg() + } + "not create response from local for payloads if header's chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val historyReader = HistoryReader.empty + + historyReader.isHeadersChainSyncedVar = false + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) + + parentActor.expectNoMsg() + } + "create response from local for payloads if header's chain is synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val historyReader = HistoryReader.empty + + historyReader.isHeadersChainSyncedVar = true + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> ids), address) + + val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + "request only unique new modifiers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isHeadersChainSyncedVar = true + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> (ids ++ blocks.map(_.id))), address) + + val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + } "process request for modifier correctly" in {} "have correct logic with check payloads to download" in {} "have correct logic with semantically successful modifier" in {} @@ -371,4 +470,20 @@ object NetworkMessagesProcessorTests extends InstanceFactory { mch -> fork } + def formHistory: (History, List[Block]) = + (0 to 10).foldLeft(generateDummyHistory(settings), List.empty[Block]) { + case ((historyMain, blocks), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 -> (blocks :+ block) + } } diff --git a/src/test/scala/encry/nvg/Utils.scala b/src/test/scala/encry/nvg/Utils.scala index 54c6ae0d34..9df1ead263 100644 --- a/src/test/scala/encry/nvg/Utils.scala +++ b/src/test/scala/encry/nvg/Utils.scala @@ -1,12 +1,10 @@ package encry.nvg import cats.Eq +import cats.syntax.eq._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import cats.instances.option._ -import cats.instances.int._ -import cats.instances.byte._ -import cats.syntax.eq._ object Utils { object instances { @@ -19,6 +17,13 @@ object Utils { override def eqv(x: OtherNodeSyncingStatus, y: OtherNodeSyncingStatus): Boolean = x.status == y.status && x.remote == y.remote } + + implicit object RequestFromLocalEq extends Eq[RequestFromLocal] { + override def eqv(x: RequestFromLocal, y: RequestFromLocal): Boolean = + x.modifierIds.size == y.modifierIds.size && + x.modifierIds.zip(y.modifierIds).forall { case (id, id1) => id === id1 } && + x.source.zip(y.source).forall { case (is1, is2) => is1 == is2 } + } } } From 9e19dd42d38465b73d818dd542eea73072d96792 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:19:31 +0300 Subject: [PATCH 101/119] added tests for request modifier, for local cache, for semantically successful modifier correctness --- .../encry/nvg/NetworkMessagesProcessor.scala | 74 +++---- .../nvg/NetworkMessagesProcessorTests.scala | 205 ++++++++++++++++-- 2 files changed, 221 insertions(+), 58 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 672154b22e..80c07af43d 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -13,7 +13,7 @@ import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryR import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ @@ -48,23 +48,20 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { - case UpdateHistoryReader(newReader: HistoryReader) => - historyReader = newReader - case SemanticallySuccessfulModifier(mod) => - //todo possible way to call CheckPayloadsToDownload - mod match { - case block: Block if historyReader.isFullChainSynced => - List(block.header, block.payload).foreach { mod: PersistentModifier => - logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") - context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) - } - modifiersRequestCache = Map( - block.encodedId -> toProto(block.header), - block.payload.encodedId -> toProto(block.payload) - ) - case _ => + case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader + + //todo possible way to call CheckPayloadsToDownload + case SemanticallySuccessfulModifier(block: Block) if historyReader.isFullChainSynced => + List(block.header, block.payload).foreach { mod: PersistentModifier => + logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") + context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) } + modifiersRequestCache = Map( + block.encodedId -> toProto(block.header), + block.payload.encodedId -> toProto(block.payload) + ) + case SemanticallySuccessfulModifier(_) => case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( @@ -94,30 +91,31 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) } - case DataFromPeer(message, remote) => - message match { - case RequestModifiersNetworkMessage((typeId, requestedIds)) => - val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds - .flatMap( - (id: ModifierId) => - modifiersRequestCache - .get(Algos.encode(id)) - .map(id -> _) - ) - .toMap - if (modifiersFromCache.nonEmpty) context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) - val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList - - typeId match { - case h if h == Header.modifierTypeId => - context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) - case _ => - getModsForRemote(unrequestedModifiers, historyReader).foreach { - case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) - } + case DataFromPeer(RequestModifiersNetworkMessage((typeId, requestedIds)), remote) => + val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds.flatMap { id: ModifierId => + modifiersRequestCache + .get(Algos.encode(id)) + .map(id -> _) + }.toMap + if (modifiersFromCache.nonEmpty) { + logger.info( + s"Send response from local with mods from cache: " + + s" ${modifiersFromCache.keys.toList.map(Algos.encode).mkString(",")} to $remote." + ) + context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) + } + val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList + + typeId match { + case h if h == Header.modifierTypeId => + context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) + case _ => + getModsForRemote(unrequestedModifiers, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) } } + case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 48f6f37492..94bbd949c5 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -2,25 +2,30 @@ package encry.nvg import java.net.InetSocketAddress -import akka.actor.{ActorRef, ActorSystem} -import akka.testkit.{TestActorRef, TestKit, TestProbe} +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ import cats.syntax.option._ -import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} +import encry.consensus.HistoryConsensus.{ Equal, Fork, Older, Unknown, Younger } import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings -import encry.view.history.{History, HistoryReader} -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage, SyncInfoNetworkMessage} +import encry.view.history.{ History, HistoryReader } +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + NetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.TaggedTypes -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scorex.utils.Random import supertagged.@@ @@ -62,7 +67,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine younger extension" in { @@ -89,7 +93,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine equals extension" in { @@ -116,7 +119,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine unknown extension" in { @@ -142,7 +144,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine fork extension" in { @@ -169,7 +170,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } } @@ -231,7 +231,7 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true } } "request only unique new modifiers" in { @@ -259,13 +259,178 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + } + "process semantically successful modifier correctly" should { + "update local cache with last semantically successful modifier" in { + val networkMessagesProcessor: TestActorRef[NetworkMessagesProcessor] = + NetworkMessagesProcessorTests.initActorState(settings) + + val reader = HistoryReader.empty + + reader.isFullChainSynced = true + + networkMessagesProcessor ! UpdateHistoryReader(reader) + + val block = generateGenesisBlock(Height @@ 1) + + networkMessagesProcessor ! SemanticallySuccessfulModifier(block) + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe true + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block.payload.encodedId) + .nonEmpty shouldBe true + + val block2 = generateGenesisBlock(Height @@ 2) + + networkMessagesProcessor ! SemanticallySuccessfulModifier(block2) + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe false + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block.payload.encodedId) + .nonEmpty shouldBe false + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block2.encodedId).nonEmpty shouldBe true + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block2.payload.encodedId) + .nonEmpty shouldBe true + } + "send broadcast message for new modifier" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val reader = HistoryReader.empty + + reader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(reader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case BroadcastModifier(modType, id) if modType == Header.modifierTypeId => + id.sameElements(block.id) shouldBe true + case BroadcastModifier(modType, id) if modType == Payload.modifierTypeId => + id.sameElements(block.payload.id) shouldBe true + } + } + } + "process request for modifier correctly" should { + "response for modifiers which are in cache by using this cache" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Header.modifierTypeId -> (blocks.headOption.get.id :: block.id :: Nil)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 1 => + ids.forall { + case (id, _) => + id.sameElements(block.id) + } shouldBe true + case ResponseFromLocal(_, _, ids) => + ids.forall { + case (id, _) => + id.sameElements(blocks.headOption.get.id) + } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => + } + } + "response for headers in 1 message for all headers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Header.modifierTypeId -> blocks.take(2).map(_.id)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 2 => + ids.keys.toList.zip(blocks.take(2).map(_.id)).forall { case (id, id1) => id.sameElements(id1) } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => } } + "response for payloads in 1 message for 1 payload" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Payload.modifierTypeId -> blocks.take(2).map(_.payload.id)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 1 => + ids.keys.toList.forall { id => + id.sameElements(blocks(1).payload.id) || id.sameElements(blocks.head.payload.id) + } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => + } + } + } + "have correct logic with check payloads to download" should { + } - "process request for modifier correctly" in {} - "have correct logic with check payloads to download" in {} - "have correct logic with semantically successful modifier" in {} } } From 532c23c7ff2f84e586de91d2f58d9d35af0abdac Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:55:20 +0300 Subject: [PATCH 102/119] added tests for check payloads to download --- .../encry/nvg/NetworkMessagesProcessor.scala | 8 +- .../nvg/NetworkMessagesProcessorTests.scala | 79 ++++++++++++++++++- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 80c07af43d..109dd8f860 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -3,7 +3,7 @@ package encry.nvg import akka.actor.{ Actor, Cancellable, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto @@ -13,7 +13,7 @@ import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryR import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ @@ -23,7 +23,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.concurrent.duration._ @@ -118,7 +118,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) - logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") + logger.debug(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 94bbd949c5..fe281a3a7e 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -6,10 +6,10 @@ import akka.actor.{ ActorRef, ActorSystem } import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ import cats.syntax.option._ -import encry.consensus.HistoryConsensus.{ Equal, Fork, Older, Unknown, Younger } +import encry.consensus.HistoryConsensus._ import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } @@ -23,13 +23,12 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ RequestModifiersNetworkMessage, SyncInfoNetworkMessage } -import org.encryfoundation.common.utils.TaggedTypes import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scorex.utils.Random -import supertagged.@@ import scala.collection.immutable +import scala.concurrent.duration._ class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -429,7 +428,79 @@ class NetworkMessagesProcessorTests } } "have correct logic with check payloads to download" should { + "request required modifiers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (_, blocks) = NetworkMessagesProcessorTests.formHistory + + val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + history.isFullChainSynced = true + + val historyReader = HistoryReader(history) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val requiredResponse = RequestFromLocal(none, Payload.modifierTypeId, blocks.take(5).map(_.payload.id)) + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true + } + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true + } + + val updHistory = blocks.drop(5).foldLeft(history) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + val updHistory2 = blocks.take(3).foldLeft(updHistory) { + case (h, block) => + h.append(block.payload).right.get._1.reportModifierIsValid(block.payload).reportModifierIsValid(block) + } + val historyReader1 = HistoryReader(updHistory2) + + networkProcessor ! UpdateHistoryReader(historyReader1) + + val requiredResponse1 = RequestFromLocal(none, Payload.modifierTypeId, blocks.drop(3).map(_.payload.id)) + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse1) shouldBe true + } + } + "not request if headers chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (_, blocks) = NetworkMessagesProcessorTests.formHistory + + val history = blocks.foldLeft(generateDummyHistory(settings)) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + history.isFullChainSynced = false + + val historyReader = HistoryReader(history) + + networkProcessor ! UpdateHistoryReader(historyReader) + + parentActor.expectMsgPF(settings.network.syncInterval + 2.seconds) { + case _: SendSyncInfo => + } + } } } } From dc805387f6643f974c008d98224b0797389a5d20 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:58:14 +0300 Subject: [PATCH 103/119] NodeViewNMProcessor tests finished --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- ...cessor.scala => NodeViewNMProcessor.scala} | 6 +- ...s.scala => NodeViewNMProcessorTests.scala} | 76 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) rename src/main/scala/encry/nvg/{NetworkMessagesProcessor.scala => NodeViewNMProcessor.scala} (96%) rename src/test/scala/encry/nvg/{NetworkMessagesProcessorTests.scala => NodeViewNMProcessorTests.scala} (90%) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index cd74130cbd..0cecfa527a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -52,7 +52,7 @@ class IntermediaryNVH( intermediaryNetwork ! RegisterForModsHandling val networkMessagesProcessor: ActorRef = - context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") + context.actorOf(NodeViewNMProcessor.props(settings), name = "Network-messages-processor") val nodeViewHolder: ActorRef = context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala similarity index 96% rename from src/main/scala/encry/nvg/NetworkMessagesProcessor.scala rename to src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 109dd8f860..77b63d1e04 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -27,7 +27,7 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.concurrent.duration._ -class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { +class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { import context.dispatcher @@ -132,6 +132,6 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St .toMap } -object NetworkMessagesProcessor { - def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) +object NodeViewNMProcessor { + def props(settings: EncryAppSettings): Props = Props(new NodeViewNMProcessor(settings)) } diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala similarity index 90% rename from src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala rename to src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala index fe281a3a7e..ddaa238e6e 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala @@ -30,7 +30,7 @@ import scorex.utils.Random import scala.collection.immutable import scala.concurrent.duration._ -class NetworkMessagesProcessorTests +class NodeViewNMProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) with WordSpecLike with Matchers @@ -45,10 +45,10 @@ class NetworkMessagesProcessorTests "determine older extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (historyMain: History, historyOlder: History) = - NetworkMessagesProcessorTests.formYoungerActorState(10, 10) + NodeViewNMProcessorTests.formYoungerActorState(10, 10) val historyReader: HistoryReader = HistoryReader(historyMain) @@ -57,7 +57,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) @@ -71,10 +71,10 @@ class NetworkMessagesProcessorTests "determine younger extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (historyMain: History, historyYounger: History) = - NetworkMessagesProcessorTests.formOlderActorState(10, 10) + NodeViewNMProcessorTests.formOlderActorState(10, 10) val historyReader: HistoryReader = HistoryReader(historyMain) @@ -83,7 +83,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) @@ -97,10 +97,10 @@ class NetworkMessagesProcessorTests "determine equals extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (history1: History, history2: History) = - NetworkMessagesProcessorTests.formEqualActorState(10, 10) + NodeViewNMProcessorTests.formEqualActorState(10, 10) val historyReader: HistoryReader = HistoryReader(history1) @@ -109,7 +109,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) @@ -123,9 +123,9 @@ class NetworkMessagesProcessorTests "determine unknown extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (h1: History, h2: History) = NetworkMessagesProcessorTests.formUnknownActorState + val (h1: History, h2: History) = NodeViewNMProcessorTests.formUnknownActorState val historyReader: HistoryReader = HistoryReader(h1) @@ -134,7 +134,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) @@ -148,10 +148,10 @@ class NetworkMessagesProcessorTests "determine fork extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (h1: History, h2: History) = - NetworkMessagesProcessorTests.formForkActorState(10, 20, 5) + NodeViewNMProcessorTests.formForkActorState(10, 20, 5) val historyReader: HistoryReader = HistoryReader(h1) @@ -160,7 +160,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) @@ -176,7 +176,7 @@ class NetworkMessagesProcessorTests "not process inv for payload while full chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -189,7 +189,7 @@ class NetworkMessagesProcessorTests "not create response from local for payloads if header's chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -210,7 +210,7 @@ class NetworkMessagesProcessorTests "create response from local for payloads if header's chain is synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -236,13 +236,13 @@ class NetworkMessagesProcessorTests "request only unique new modifiers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -264,8 +264,8 @@ class NetworkMessagesProcessorTests } "process semantically successful modifier correctly" should { "update local cache with last semantically successful modifier" in { - val networkMessagesProcessor: TestActorRef[NetworkMessagesProcessor] = - NetworkMessagesProcessorTests.initActorState(settings) + val networkMessagesProcessor: TestActorRef[NodeViewNMProcessor] = + NodeViewNMProcessorTests.initActorState(settings) val reader = HistoryReader.empty @@ -301,7 +301,7 @@ class NetworkMessagesProcessorTests "send broadcast message for new modifier" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val reader = HistoryReader.empty @@ -326,9 +326,9 @@ class NetworkMessagesProcessorTests "response for modifiers which are in cache by using this cache" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -365,9 +365,9 @@ class NetworkMessagesProcessorTests "response for headers in 1 message for all headers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -396,9 +396,9 @@ class NetworkMessagesProcessorTests "response for payloads in 1 message for 1 payload" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -431,9 +431,9 @@ class NetworkMessagesProcessorTests "request required modifiers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (_, blocks) = NetworkMessagesProcessorTests.formHistory + val (_, blocks) = NodeViewNMProcessorTests.formHistory val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { case (h, block) => @@ -482,9 +482,9 @@ class NetworkMessagesProcessorTests "not request if headers chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (_, blocks) = NetworkMessagesProcessorTests.formHistory + val (_, blocks) = NodeViewNMProcessorTests.formHistory val history = blocks.foldLeft(generateDummyHistory(settings)) { case (h, block) => @@ -505,11 +505,11 @@ class NetworkMessagesProcessorTests } } -object NetworkMessagesProcessorTests extends InstanceFactory { +object NodeViewNMProcessorTests extends InstanceFactory { - def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NetworkMessagesProcessor] = { - val networkProcessor: TestActorRef[NetworkMessagesProcessor] = - TestActorRef[NetworkMessagesProcessor](NetworkMessagesProcessor.props(settings)) + def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NodeViewNMProcessor] = { + val networkProcessor: TestActorRef[NodeViewNMProcessor] = + TestActorRef[NodeViewNMProcessor](NodeViewNMProcessor.props(settings)) networkProcessor } From 0fa0a3da1254727d4459bfebd7caf2fec382bacd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 20:11:41 +0300 Subject: [PATCH 104/119] fixed proto parsing error finished tests for modifiers validator --- src/main/scala/encry/network/BlackList.scala | 1 + .../scala/encry/nvg/IntermediaryNVH.scala | 4 +- .../scala/encry/nvg/ModifiersValidator.scala | 83 ++++--- .../encry/view/history/HistoryReader.scala | 4 +- .../DownloadedModifiersValidatorTests.scala | 231 ------------------ src/test/scala/encry/nvg/ValidatorTests.scala | 181 +++++++++++++- 6 files changed, 235 insertions(+), 269 deletions(-) delete mode 100644 src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala diff --git a/src/main/scala/encry/network/BlackList.scala b/src/main/scala/encry/network/BlackList.scala index a360757969..0d84bb4f02 100644 --- a/src/main/scala/encry/network/BlackList.scala +++ b/src/main/scala/encry/network/BlackList.scala @@ -38,6 +38,7 @@ object BlackList { case object SyntacticallyInvalidPersistentModifier extends BanReason case object SyntacticallyInvalidTransaction extends BanReason case object CorruptedSerializedBytes extends BanReason + case object ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage extends BanReason case object SpamSender extends BanReason case object SentPeersMessageWithoutRequest extends BanReason case object SentInvForPayload extends BanReason diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 0cecfa527a..a1585f5e7f 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -58,7 +58,7 @@ class IntermediaryNVH( val modifiersValidatorRouter: ActorRef = context.actorOf( BalancingPool(5) - .props(ModifiersValidator.props(nodeViewHolder, settings)), + .props(ModifiersValidator.props(nodeViewHolder, self, settings)), name = "Modifiers-validator-router" ) val snapshotProcessor: Option[ActorRef] = @@ -98,7 +98,7 @@ class IntermediaryNVH( case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e3990ec576..4814b7736f 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -11,6 +11,7 @@ import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidatio import encry.modifiers.history.{ HeaderUtils, PayloadUtils } import encry.network.BlackList.BanReason.{ CorruptedSerializedBytes, + ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } @@ -24,37 +25,68 @@ import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSeriali import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { +import scala.util.Try + +class ModifiersValidator( + nodeViewHolderRef: ActorRef, + intermediaryNVH: ActorRef, + settings: EncryAppSettings +) extends Actor + with StrictLogging { override def receive: Receive = { - case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => - fromProto(modifierTypeId, modifierBytes) match { + case ModifierForValidation(reader, id, modifierTypeId, modifierBytes, remote) if !reader.isModifierDefined(id) => + ModifiersValidator.fromProto(modifierTypeId, modifierBytes) match { case Left(error) => - logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") - context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifierBytes(modifierId) + logger.info(s"Modifier ${Algos.encode(id)} is incorrect cause: ${error.getMessage}.") + intermediaryNVH ! BanPeer(remote, CorruptedSerializedBytes) + intermediaryNVH ! InvalidModifierBytes(id) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = - isPreSemanticValid(modifier, reader, settings) + ModifiersValidator.isPreSemanticValid(modifier, reader, settings) val syntacticValidation: Boolean = - isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) + ModifiersValidator.isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { - logger.debug(s"Modifier ${modifier.encodedId} is valid.") - nodeViewHolderRef ! ValidatedModifier(modifier) + if (modifier.id.sameElements(id)) { + logger.debug(s"Modifier ${modifier.encodedId} is valid.") + nodeViewHolderRef ! ValidatedModifier(modifier) + } else { + logger.info(s"Modifier ${modifier.encodedId} should have ${Algos.encode(id)} id!") + intermediaryNVH ! BanPeer(remote, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) + } } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") - context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! SyntacticallyFailedModification(modifier, List.empty) + intermediaryNVH ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") - context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) - context.parent ! SyntacticallyFailedModification(modifier, List.empty) + intermediaryNVH ! BanPeer(remote, PreSemanticInvalidModifier(error)) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) } } + case m: ModifierForValidation => + logger.info(s"Got modifier ${Algos.encode(m.modifierId)} but this mod is already in history.") } +} + +object ModifiersValidator { + + final case class ModifierForValidation( + historyReader: HistoryReader, + modifierId: ModifierId, + modifierTypeId: ModifierTypeId, + modifierBytes: Array[Byte], + remote: InetSocketAddress + ) + + final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + + final case class InvalidModifierBytes(id: ModifierId) extends AnyVal + private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, @@ -77,8 +109,8 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings bytes: Array[Byte] ): Either[Throwable, PersistentModifier] = Either.fromTry(modType match { - case Header.modifierTypeId => HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes)) - case Payload.modifierTypeId => PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes)) + case Header.modifierTypeId => Try(HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes))).flatten + case Payload.modifierTypeId => Try(PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes))).flatten }) def isSyntacticallyValid( @@ -88,22 +120,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case h: Header => HeaderUtils.syntacticallyValidity(h, modifierIdSize).isSuccess case p: Payload => PayloadUtils.syntacticallyValidity(p, modifierIdSize).isSuccess } -} - -object ModifiersValidator { - - final case class ModifierForValidation( - historyReader: HistoryReader, - modifierId: ModifierId, - modifierTypeId: ModifierTypeId, - modifierBytes: Array[Byte], - remote: InetSocketAddress - ) - - final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal - - final case class InvalidModifierBytes(id: ModifierId) extends AnyVal - def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = - Props(new ModifiersValidator(nodeViewHolderRef, settings)) + def props(nodeViewHolderRef: ActorRef, intermediaryNVH: ActorRef, settings: EncryAppSettings): Props = + Props(new ModifiersValidator(nodeViewHolderRef, intermediaryNVH, settings)) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 64f9c9edb7..7d781b3f00 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,9 +1,9 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import encry.modifiers.history.HeaderChain import io.iohk.iodb.ByteArrayWrapper -import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala deleted file mode 100644 index 0b05c5bfb7..0000000000 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ /dev/null @@ -1,231 +0,0 @@ -package encry.network - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{ TestActorRef, TestProbe } -import encry.modifiers.InstanceFactory -import encry.network.BlackList.BanReason._ -//import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, UpdatedHistory } -import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } -import encry.network.PeersKeeper.BanPeer -import encry.settings.TestNetSettings -//import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote -import encry.view.history.History -import org.encryfoundation.common.crypto.equihash.EquihashSolution -import org.encryfoundation.common.modifiers.history.{ - Block, - Header, - HeaderProtoSerializer, - Payload, - PayloadProtoSerializer -} -import org.encryfoundation.common.network.BasicMessagesRepo.Handshake -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } -import scorex.crypto.hash.Digest32 -import scorex.utils.Random - -//class DownloadedModifiersValidatorTests -// extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem() -// -// override def afterAll(): Unit = system.terminate() -// -// "DownloadedModifiersValidatorTests" should { -// "find too old header by height" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) -// downloadedModifiersValidator ! UpdatedHistory(history) -// val invalidHeader = generateGenesisBlock(Height @@ 1) -// -// val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) -// .map( -// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray -// ) -// .toMap -// import scala.concurrent.duration._ -// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) -// peersKeeper.expectMsgPF(10.seconds) { -// case BanPeer(connected, _) => connected == connectedPeer -// } -// val validHeightHeader = generateGenesisBlock(Height @@ 200) -// -// val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) -// .map( -// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray -// ) -// .toMap -// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) -// nodeViewHolder.expectMsgPF(10.seconds) { -// case ModifierFromRemote(mod) => mod == validHeightHeader.header -// } -// } -// "find corrupted header" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val deliveryManager = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val history: History = generateDummyHistory(testNetSettings) -// -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val timestamp1 = System.currentTimeMillis() -// Thread.sleep(1000) -// val timestamp2 = System.currentTimeMillis() -// -// val header_first: Header = Header( -// 1.toByte, -// ModifierId @@ Random.randomBytes(), -// Digest32 @@ Random.randomBytes(), -// timestamp2, -// 2, -// scala.util.Random.nextLong(), -// testNetSettings.constants.InitialDifficulty, -// EquihashSolution(Seq(1, 3)), -// Random.randomBytes() -// ) -// val header_second: Header = Header( -// 1.toByte, -// header_first.id, -// Digest32 @@ Random.randomBytes(), -// timestamp1, -// 1, -// scala.util.Random.nextLong(), -// testNetSettings.constants.InitialDifficulty, -// EquihashSolution(Seq(1, 3)), -// Random.randomBytes() -// ) -// -// history.append(header_first) -// -// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) -// -// /* Header */ -// val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap -// val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) -// -// deliveryManager.send(downloadedModifiersValidator, msg) -// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) -// nodeViewHolder.expectNoMsg() -// nodeViewSync.expectMsg(InvalidModifier(header_second.id)) -// } -// "find corrupted payload" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val deliveryManager = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val history: History = generateDummyHistory(testNetSettings) -// -// val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// (prevHistory.reportModifierIsValid(block), blocks :+ block) -// } -// -// val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) -// -// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) -// -// val bytes = PayloadProtoSerializer.toProto(payload).toByteArray -// -// val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( -// b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse -// ) :+ (payload.id -> bytes)).toMap -// -// deliveryManager -// .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) -// -// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) -// nodeViewHolder.expectMsg(ModifierFromRemote(payload)) -// } -// } -// -// def generateBlocks(qty: Int, history: History): (History, List[Block]) = -// (0 until qty).foldLeft(history, List.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// val a = prevHistory.reportModifierIsValid(block) -// (a, blocks :+ block) -// } -//} diff --git a/src/test/scala/encry/nvg/ValidatorTests.scala b/src/test/scala/encry/nvg/ValidatorTests.scala index 72bcac2eed..83af8da7f7 100644 --- a/src/test/scala/encry/nvg/ValidatorTests.scala +++ b/src/test/scala/encry/nvg/ValidatorTests.scala @@ -1,5 +1,184 @@ package encry.nvg -class ValidatorTests { +import java.net.InetSocketAddress + +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestKit, TestProbe } +import encry.modifiers.InstanceFactory +import encry.network.BlackList.BanReason.{ + CorruptedSerializedBytes, + ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, + PreSemanticInvalidModifier, + SyntacticallyInvalidPersistentModifier +} +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{ Block, Header, HeaderProtoSerializer } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +import scorex.utils.Random + +class ValidatorTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest { + + override def afterAll(): Unit = system.terminate() + + "Modifiers validator" should { + "notify intermediary actor about modifier with invalid raw bytes" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBytes = Random.randomBytes(190) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val randomId = ModifierId @@ Random.randomBytes() + + networkProcessor ! ModifierForValidation(reader, randomId, Header.modifierTypeId, corruptedBytes, remote) + + intermediary.expectMsgPF() { + case BanPeer(r, CorruptedSerializedBytes) => r shouldBe remote + case InvalidModifierBytes(id) => id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary actor about pre semantic invalid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBlock = blocks.head.copy( + header = blocks.head.header.copy(height = -1000) + ) + + val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, PreSemanticInvalidModifier(_)) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary actor about syntactically invalid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBlock = blocks.head.copy( + header = blocks.head.header.copy(parentId = ModifierId @@ blocks.head.header.id.drop(2)) + ) + + val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, SyntacticallyInvalidPersistentModifier) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary about incorrect modifier id" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify nvh actor about valid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, _) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val correctBlock: Block = generateNextBlock(history) + + val correctBytes = HeaderProtoSerializer.toProto(correctBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, correctBlock.id, Header.modifierTypeId, correctBytes, remote) + + nvh.expectMsgPF() { + case ValidatedModifier(mod) => mod.id.sameElements(correctBlock.id) shouldBe true + } + } + } } From f793d13dbcfae0d7e782882fcec143c9d8fec8bc Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 09:26:19 +0300 Subject: [PATCH 105/119] add test for pk pipeline --- src/main/scala/encry/network/PK.scala | 28 +- .../network/ConnectWithNewPeerTests.scala | 306 +++++++++--------- 2 files changed, 174 insertions(+), 160 deletions(-) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 3c2efb1c5a..9d84c09143 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -33,8 +33,8 @@ class PK(networkSettings: NetworkSettings, var blackList: BlackList = BlackList(blacklistSettings) - var knownPeers: Set[InetAddress] = networkSettings.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet + var knownPeers: Set[InetSocketAddress] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer }.toSet var outgoingConnections: Set[InetSocketAddress] = Set.empty @@ -48,15 +48,6 @@ class PK(networkSettings: NetworkSettings, PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" ), self) - context.system.scheduler.schedule(5.seconds, 5.seconds) { - context.parent ! UpdatingPeersInfo( - peersForConnection.keys.toList, - connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => - (peer.socketAddress, peer.handshake.nodeName, peer.direction) - ).toList, - blackList.getAll.toList - ) - } context.system.scheduler.schedule(600.millis, blacklistSettings.cleanupTime){blackList = blackList.cleanupBlackList} } @@ -100,14 +91,16 @@ class PK(networkSettings: NetworkSettings, outgoingConnections -= remote sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) } - else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { + else if (connectWithOnlyKnownPeers && knownPeers.contains(remote)) { logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") + awaitingHandshakeConnections += remote sender() ! ConnectionVerified(remote, remoteConnection, Incoming) } else if (connectWithOnlyKnownPeers) logger.info(s"Got incoming connection but we can connect only with known peers.") else { logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") + awaitingHandshakeConnections += remote sender() ! ConnectionVerified(remote, remoteConnection, Incoming) } } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + @@ -126,7 +119,7 @@ class PK(networkSettings: NetworkSettings, peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + s" Current collection is: ${peersForConnection.keys.mkString(",")}.") - + updatePeersCollection() case ConnectionStopped(peer) => logger.info(s"Connection stopped for: $peer.") awaitingHandshakeConnections -= peer @@ -136,6 +129,7 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } + updatePeersCollection() case predicate: GetPeerByPredicate => connectedPeers.getAll.find { case (_, info) => predicate.predicate(info) }.map { @@ -197,6 +191,14 @@ class PK(networkSettings: NetworkSettings, InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) + def updatePeersCollection(): Unit = context.parent ! UpdatingPeersInfo( + peersForConnection.keys.toList, + connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => + (peer.socketAddress, peer.handshake.nodeName, peer.direction) + ).toList, + blackList.getAll.toList + ) + def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer } diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 5f8160f949..3f40526557 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -2,164 +2,176 @@ package encry.network import java.net.InetSocketAddress -import akka.actor.ActorSystem +import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestActorRef, TestProbe} +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.network.PeersKeeper._ import encry.settings.TestNetSettings -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, PeersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, PeersNetworkMessage} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} + import scala.concurrent.duration._ -//class ConnectWithNewPeerTests extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { +class ConnectWithNewPeerTests extends WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem() + + override def afterAll(): Unit = system.terminate() + + val knowPeersSettings = testNetSettings.copy( + network = testNetSettings.network.copy( + knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), + connectOnlyWithKnownPeers = Some(true) + ), + blackList = testNetSettings.blackList.copy( + banTime = 2 seconds, + cleanupTime = 3 seconds + )) + + "Peers keeper" should { + "maintain outgoing connection process correctly" in { + /* Request first peer while current number of connections is 0 */ + val networkController: TestProbe = TestProbe() + val nodeViewSync = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + + val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers + + networkController.send(peersKeeper, RequestPeerForConnection) + println(peersKeeper.underlyingActor.knownPeers.map(PeerForConnection)) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val msg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe true + + val remoteAkkaConnectionHandler = TestProbe() + + networkController.send(peersKeeper, NewConnection(msg.peer, remoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, remoteAkkaConnectionHandler.ref, Outgoing)) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe true + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer", Some(availablePeers.head), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head) shouldBe true + networkController.expectMsgType[UpdatingPeersInfo] + + /* Request next peer after first connection setup */ + + val newAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.drop(1) + + networkController.send(peersKeeper, RequestPeerForConnection) + val nextPeerForConnectionMsg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(nextPeerForConnectionMsg.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeerForConnectionMsg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, remoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, remoteAkkaConnectionHandler.ref, Incoming)) + peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe true + + val newPeerHandler = TestProbe() + val newConnectedPeer: ConnectedPeer = ConnectedPeer(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, newPeerHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer_new", Some(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe true + networkController.expectMsgType[UpdatingPeersInfo] + /* Try to ask one more peer while max number of connections has been expired */ + + networkController.send(peersKeeper, RequestPeerForConnection) + networkController.expectNoMsg() + + /* Now we will ban one peer */ + + val actorWhichSendBanMessage = TestProbe() + + actorWhichSendBanMessage.send(peersKeeper, BanPeer(newConnectedPeer.socketAddress, ExpiredNumberOfConnections)) + newPeerHandler.expectMsgAnyOf(CloseConnection) + peersKeeper.underlyingActor.blackList.contains(newConnectedPeer.socketAddress.getAddress) shouldBe true + networkController.send(peersKeeper, ConnectionStopped(newConnectedPeer.socketAddress)) + peersKeeper.underlyingActor.connectedPeers.contains(newConnectedPeer.socketAddress) shouldBe false + networkController.expectMsgType[UpdatingPeersInfo] + + /* Try to setup Incoming connection from banned peer */ + + networkController.send(peersKeeper, NewConnection(newConnectedPeer.socketAddress, remoteAkkaConnectionHandler.ref)) + networkController.expectNoMsg() + + /* Try to request new connection */ + + val updatedAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.filter(peer => + peer != msg.peer && peer != newConnectedPeer.socketAddress + ) + + networkController.send(peersKeeper, RequestPeerForConnection) + val oneMorePeer = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe true + + val updatedRemoteAkkaConnectionHandler = TestProbe() + + networkController.send(peersKeeper, NewConnection(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref, Outgoing)) + peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe true + + val updatedConnectedPeer: ConnectedPeer = ConnectedPeer(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer", Some(oneMorePeer.peer), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(updatedConnectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(oneMorePeer.peer) shouldBe true + } +// "remove peer from available we can't connect to" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers // -// implicit val system: ActorSystem = ActorSystem() +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) // -// override def afterAll(): Unit = system.terminate() +// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) +// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) // -// val knowPeersSettings = testNetSettings.copy( -// network = testNetSettings.network.copy( -// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), -// connectOnlyWithKnownPeers = Some(true) -// ), -// blackList = testNetSettings.blackList.copy( -// banTime = 2 seconds, -// cleanupTime = 3 seconds -// )) +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) // -// "Peers keeper" should { - // "maintain outgoing connection process correctly" in { - // /* Request first peer while current number of connections is 0 */ - // val networkController = TestProbe() - // val nodeViewSync = TestProbe() - // val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) - // - // val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe true - // - // val remoteAkkaConnectionHandler = TestProbe() - // - // networkController.send(peersKeeper, VerifyConnection(availablePeers.head._1, remoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(availablePeers.head._1, remoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe true - // - // val peerHandler = TestProbe() - // val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(0) - // - // /* Request next peer after first connection setup */ - // - // val newAvailablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers.drop(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(newAvailablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe true - // - // networkController.send(peersKeeper, VerifyConnection(newAvailablePeers.head._1, remoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(newAvailablePeers.head._1, remoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe true - // - // val newPeerHandler = TestProbe() - // val newConnectedPeer: ConnectedPeer = ConnectedPeer(newAvailablePeers.head._1, newPeerHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer_new", Some(newAvailablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(newAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(newAvailablePeers.head._1) shouldBe Some(0) - // - // /* Try to ask one more peer while max number of connections has been expired */ - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectNoMsg() - // - // /* Now we will ban one peer */ - // - // val actorWhichSendBanMessage = TestProbe() - // - // actorWhichSendBanMessage.send(peersKeeper, BanPeer(newConnectedPeer, ExpiredNumberOfConnections)) - // newPeerHandler.expectMsgAnyOf(CloseConnection, GetPeersNetworkMessage) - // peersKeeper.underlyingActor.blackList.contains(newConnectedPeer.socketAddress.getAddress) shouldBe true - // networkController.send(peersKeeper, ConnectionStopped(newConnectedPeer.socketAddress)) - // peersKeeper.underlyingActor.knownPeers.contains(newConnectedPeer.socketAddress) shouldBe false - // peersKeeper.underlyingActor.connectedPeers.contains(newConnectedPeer.socketAddress) shouldBe false - // - // /* Try to setup Incoming connection from banned peer */ - // - // networkController.send(peersKeeper, VerifyConnection(newConnectedPeer.socketAddress, remoteAkkaConnectionHandler.ref)) - // networkController.expectNoMsg() - // - // /* Try to request new connection */ - // - // val updatedAvailablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers.takeRight(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(updatedAvailablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // - // val updatedRemoteAkkaConnectionHandler = TestProbe() - // - // networkController.send(peersKeeper, VerifyConnection(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(updatedAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // - // val updatedConnectedPeer: ConnectedPeer = ConnectedPeer(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer", Some(updatedAvailablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(updatedConnectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(updatedAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(updatedAvailablePeers.head._1) shouldBe Some(0) - // } - // "remove peer from available we can't connect to" in { - // val networkController = TestProbe() - // val nodeViewSync = TestProbe() - // val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) - // - // val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // - // networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // - // networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true - // } +// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) +// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true +// } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() // val nodeViewSync = TestProbe() @@ -340,5 +352,5 @@ import scala.concurrent.duration._ // networkController.expectMsg( // ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) // } -// } -//} \ No newline at end of file + } +} \ No newline at end of file From 117dc271c2ac59fc26d2d94d2bac68f18eeb8f07 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:12:25 +0300 Subject: [PATCH 106/119] stats added --- .../scala/encry/nvg/ModifiersValidator.scala | 2 + .../scala/encry/nvg/NodeViewNMProcessor.scala | 2 +- src/main/scala/encry/stats/StatsSender.scala | 149 ++++++++++-------- 3 files changed, 87 insertions(+), 66 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 4814b7736f..d91a97e66b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -19,6 +19,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings +import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } @@ -49,6 +50,7 @@ class ModifiersValidator( if (preSemanticValidation.isRight && syntacticValidation) { if (modifier.id.sameElements(id)) { logger.debug(s"Modifier ${modifier.encodedId} is valid.") + intermediaryNVH ! ValidatedModifierFromNetwork(modifierTypeId) nodeViewHolderRef ! ValidatedModifier(modifier) } else { logger.info(s"Modifier ${modifier.encodedId} should have ${Algos.encode(id)} id!") diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 77b63d1e04..693cab0618 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -65,7 +65,7 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( - s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + + s"\n\nComparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + s"Comparison result is $comparison.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) diff --git a/src/main/scala/encry/stats/StatsSender.scala b/src/main/scala/encry/stats/StatsSender.scala index deae07515f..3295f2c3c1 100644 --- a/src/main/scala/encry/stats/StatsSender.scala +++ b/src/main/scala/encry/stats/StatsSender.scala @@ -4,27 +4,29 @@ import java.io.File import java.net.InetAddress import java.util import java.text.SimpleDateFormat -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp.timeProvider import encry.consensus.EncrySupplyController -import encry.settings.{InfluxDBSettings, NetworkSettings} +import encry.settings.{ InfluxDBSettings, NetworkSettings } import encry.stats.StatsSender._ import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId, ModifierTypeId } import org.encryfoundation.common.utils.constants.Constants -import org.influxdb.{InfluxDB, InfluxDBFactory} +import org.influxdb.{ InfluxDB, InfluxDBFactory } import scala.concurrent.ExecutionContext.Implicits.global -class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants) extends Actor with StrictLogging { +class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants) + extends Actor + with StrictLogging { var modifiersToDownload: Map[String, (ModifierTypeId, Long)] = Map.empty - var modifiersToApply: Map[String, (ModifierTypeId, Long)] = Map.empty + var modifiersToApply: Map[String, (ModifierTypeId, Long)] = Map.empty val nodeName: String = networkSettings.nodeName match { case Some(value) => value - case None => InetAddress.getLocalHost.getHostAddress + ":" + networkSettings.bindAddress.getPort + case None => InetAddress.getLocalHost.getHostAddress + ":" + networkSettings.bindAddress.getPort } val influxDB: InfluxDB = InfluxDBFactory .connect(influxDBSettings.url, influxDBSettings.login, influxDBSettings.password) @@ -47,7 +49,8 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe s"value=${new File("encry/data/history/").listFiles.foldLeft(0L)(_ + _.length())}", //++ s"supply,nodeName=$nodeName,height=${fb.height} " + s"value=${EncrySupplyController.supplyAt(fb.height.asInstanceOf[Height], constants)}" //++ - )) + ) + ) case HeightStatistics(bestHeaderHeight, bestBlockHeight) => influxDB.write( @@ -75,33 +78,41 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe case InfoAboutTransactionsFromMiner(qty) => influxDB.write(influxDBSettings.udpPort, s"infoAboutTxsFromMiner,nodeName=$nodeName value=$qty") - case GetModifiers(_, modifiers) => modifiers - .foreach(downloadedModifierId => - modifiersToDownload.get(Algos.encode(downloadedModifierId)).foreach { dowloadInfo => - influxDB.write( - influxDBSettings.udpPort, - s"modDownloadStat,nodeName=$nodeName,modId=${Algos.encode(downloadedModifierId)}," + - s"modType=${dowloadInfo._1} value=${System.currentTimeMillis() - dowloadInfo._2}" - ) - modifiersToDownload = modifiersToDownload - Algos.encode(downloadedModifierId) - } - ) + case GetModifiers(_, modifiers) => + modifiers + .foreach( + downloadedModifierId => + modifiersToDownload.get(Algos.encode(downloadedModifierId)).foreach { dowloadInfo => + influxDB.write( + influxDBSettings.udpPort, + s"modDownloadStat,nodeName=$nodeName,modId=${Algos.encode(downloadedModifierId)}," + + s"modType=${dowloadInfo._1} value=${System.currentTimeMillis() - dowloadInfo._2}" + ) + modifiersToDownload = modifiersToDownload - Algos.encode(downloadedModifierId) + } + ) - case MiningEnd(blockHeader, workerIdx, workersQty) => timeProvider - .time() - .map(time => influxDB.write( - influxDBSettings.udpPort, - util.Arrays.asList( - s"miningEnd,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + - s"height=${blockHeader.height},worker=$workerIdx value=${time - blockHeader.timestamp}", - s"minerIterCount,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + - s"height=${blockHeader.height} value=${blockHeader.nonce - Long.MaxValue / workersQty * workerIdx + 1}" - ))) + case MiningEnd(blockHeader, workerIdx, workersQty) => + timeProvider + .time() + .map( + time => + influxDB.write( + influxDBSettings.udpPort, + util.Arrays.asList( + s"miningEnd,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + + s"height=${blockHeader.height},worker=$workerIdx value=${time - blockHeader.timestamp}", + s"minerIterCount,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + + s"height=${blockHeader.height} value=${blockHeader.nonce - Long.MaxValue / workersQty * workerIdx + 1}" + ) + ) + ) case EndOfApplyingModifier(modifierId) => modifiersToApply.get(Algos.encode(modifierId)).foreach { modInfo => - influxDB.write(influxDBSettings.udpPort, s"modifApplying,nodeName=$nodeName," + - s"modType=${modInfo._1} value=${System.currentTimeMillis() - modInfo._2}") + influxDB.write(influxDBSettings.udpPort, + s"modifApplying,nodeName=$nodeName," + + s"modType=${modInfo._1} value=${System.currentTimeMillis() - modInfo._2}") modifiersToApply -= Algos.encode(modifierId) } @@ -112,65 +123,73 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe case SleepTime(time) => influxDB.write(influxDBSettings.udpPort, s"sleepTime,nodeName=$nodeName value=$time") - case StateUpdating(time) => influxDB.write(influxDBSettings.udpPort, s"stateUpdatingTime,nodeName=$nodeName value=$time") + case StateUpdating(time) => + influxDB.write(influxDBSettings.udpPort, s"stateUpdatingTime,nodeName=$nodeName value=$time") case AvlStat(storageInsert: Long, avlDeleteTime: Long, avlInsertTime: Long) => - influxDB.write(influxDBSettings.udpPort, + influxDB.write( + influxDBSettings.udpPort, s"avlStat,nodeName=$nodeName avlDelete=$avlDeleteTime,insertTime=$avlInsertTime,value=$storageInsert" ) case UtxoStat(txsNumber: Int, validationTime: Long) => - influxDB.write(influxDBSettings.udpPort, s"utxoStat,nodeName=$nodeName txsNumber=$txsNumber,value=$validationTime") - - case msg: ModifiersDownloadStatistic => msg match { - case _ if nodeName.exists(_.isDigit) => - val nodeNumber: Long = nodeName.filter(_.isDigit).toLong - val (isHeader: Boolean, tableName: String) = msg match { - case SerializedModifierFromNetwork(t) => - (t == Header.modifierTypeId) -> "serializedModifierFromNetwork" - case ValidatedModifierFromNetwork(t) => - (t == Header.modifierTypeId) -> "validatedModifierFromNetwork" - } - influxDB.write( - influxDBSettings.udpPort, - s"""$tableName,nodeName=$nodeNumber,isHeader=$isHeader value=$nodeNumber""" - ) - case _ => //do nothing - } + influxDB.write(influxDBSettings.udpPort, + s"utxoStat,nodeName=$nodeName txsNumber=$txsNumber,value=$validationTime") + + case msg: ModifiersDownloadStatistic => + msg match { + case _ if nodeName.exists(_.isDigit) => + val nodeNumber: Long = nodeName.filter(_.isDigit).toLong + val (isHeader: Boolean, tableName: String) = msg match { + case SerializedModifierFromNetwork(t) => + (t == Header.modifierTypeId) -> "serializedModifierFromNetwork" + case ValidatedModifierFromNetwork(t) => + (t == Header.modifierTypeId) -> "validatedModifierFromNetwork" + } + influxDB.write( + influxDBSettings.udpPort, + s"""$tableName,nodeName=$nodeNumber,isHeader=$isHeader value=$nodeNumber""" + ) + case _ => //do nothing + } case ModifierAppendedToHistory(_, _) => - case ModifierAppendedToState(_) => - + case ModifierAppendedToState(_) => case SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) => - modifiersToDownload = modifiersToDownload ++ modifiers.map(mod => (Algos.encode(mod), (modifierTypeId, System.currentTimeMillis()))) + modifiersToDownload = modifiersToDownload ++ modifiers.map( + mod => (Algos.encode(mod), (modifierTypeId, System.currentTimeMillis())) + ) } } object StatsSender { sealed trait StatsSenderMessage - final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage - final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage - final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage + final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage + final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage + final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) extends StatsSenderMessage - final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage - final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal - final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage - final case class StateUpdating(time: Long) extends StatsSenderMessage - final case class SleepTime(time: Long) extends AnyVal - final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) extends StatsSenderMessage + final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage + final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal + final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage + final case class StateUpdating(time: Long) extends StatsSenderMessage + final case class SleepTime(time: Long) extends AnyVal + final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) + extends StatsSenderMessage final case class MiningEnd(blockHeader: Header, workerIdx: Int, workersQty: Int) final case class MiningTime(time: Long) extends AnyVal final case class SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) final case class GetModifiers(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) sealed trait ModifiersDownloadStatistic final case class SerializedModifierFromNetwork(modifierTypeId: ModifierTypeId) extends ModifiersDownloadStatistic - final case class ValidatedModifierFromNetwork(modifierTypeId: ModifierTypeId) extends ModifiersDownloadStatistic + final case class ValidatedModifierFromNetwork(modifierTypeId: ModifierTypeId) + extends ModifiersDownloadStatistic + with StatsSenderMessage final case class AvlStat(storageInsert: Long, avlDeleteTime: Long, avlInsertTime: Long) final case class UtxoStat(txsNumber: Int, validationTime: Long) final case class NewHeightByHistory(height: Int) extends AnyVal - final case class NewHeightByState(height: Int) extends AnyVal + final case class NewHeightByState(height: Int) extends AnyVal def props(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants): Props = Props(new StatsSender(influxDBSettings, networkSettings, constants)) -} \ No newline at end of file +} From 73f3357dd19f68d640c508b869ae6e588f58cc3b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:12:51 +0300 Subject: [PATCH 107/119] fixed modifiers propagation --- src/main/scala/encry/nvg/ModifiersValidator.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index d91a97e66b..94ef506adf 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -25,7 +25,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } - import scala.util.Try class ModifiersValidator( From c427b7258e6d4108e81c7dd080f59325f4ac2a5c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:18:05 +0300 Subject: [PATCH 108/119] add NotifyNodeAboutModifier --- .../scala/encry/network/MessageBuilder.scala | 1 - src/main/scala/encry/network/Messages.scala | 3 + .../network/ConnectWithNewPeerTests.scala | 56 +++++++++---------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 3adeba3341..873628cfb4 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -77,7 +77,6 @@ case class MessageBuilder(peersKeeper: ActorRef, context.parent ! MsgSent(PeersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } - } } diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index a9303937eb..2e47bb1646 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -19,5 +19,8 @@ object Messages { modifierId: ModifierId) extends MessageToNetwork final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork + final case class NotifyNodeAboutModifier(source: InetSocketAddress, + modifierTypeId: ModifierTypeId, + modifierId: ModifierId) extends MessageToNetwork } } diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 3f40526557..a76153a211 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -10,7 +10,7 @@ import encry.network.BlackList.BanReason._ import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} -import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper._ import encry.settings.TestNetSettings import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, PeersNetworkMessage} @@ -43,7 +43,6 @@ class ConnectWithNewPeerTests extends WordSpecLike "maintain outgoing connection process correctly" in { /* Request first peer while current number of connections is 0 */ val networkController: TestProbe = TestProbe() - val nodeViewSync = TestProbe() val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( PK.props(settings.network.copy(maxConnections = 2), settings.blackList), networkController.ref @@ -124,10 +123,6 @@ class ConnectWithNewPeerTests extends WordSpecLike /* Try to request new connection */ - val updatedAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.filter(peer => - peer != msg.peer && peer != newConnectedPeer.socketAddress - ) - networkController.send(peersKeeper, RequestPeerForConnection) val oneMorePeer = networkController.expectMsgType[PeerForConnection] peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe true @@ -148,30 +143,31 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe false peersKeeper.underlyingActor.knownPeers.contains(oneMorePeer.peer) shouldBe true } -// "remove peer from available we can't connect to" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) -// -// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) -// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) -// -// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) -// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true -// } + "remove peer from available we can't connect to" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + + val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers + + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + + networkController.send(peersKeeper, OutgoingConnectionFailed(msg.peer)) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe false + + networkController.send(peersKeeper, RequestPeerForConnection) + val nextPeer = networkController.expectMsgType[PeerForConnection] + + networkController.send(peersKeeper, OutgoingConnectionFailed(nextPeer.peer)) + peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true + } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() // val nodeViewSync = TestProbe() From 913230ee5dd177dc618ffe280468f468ffeea7da Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:19:16 +0300 Subject: [PATCH 109/119] ec-fix --- src/main/scala/encry/network/Messages.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 2e47bb1646..0f7c0f1641 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -21,6 +21,6 @@ object Messages { final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork final case class NotifyNodeAboutModifier(source: InetSocketAddress, modifierTypeId: ModifierTypeId, - modifierId: ModifierId) extends MessageToNetwork + modifierId: List[ModifierId]) extends MessageToNetwork } } From 9cea1f81efad826022e29c104c6c1bc896bdcb4b Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:21:15 +0300 Subject: [PATCH 110/119] add InvNetworkMessage on message builder --- src/main/scala/encry/network/MessageBuilder.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 873628cfb4..8dc72f1897 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -10,7 +10,7 @@ import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, NotifyNodeAboutModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -63,13 +63,18 @@ case class MessageBuilder(peersKeeper: ActorRef, context.parent ! MsgSent(ModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } - context.stop(self) + case NotifyNodeAboutModifier(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! InvNetworkMessage(modTypeId -> modsIds) + context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peer.socketAddress) + } + } case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } - context.stop(self) case SendPeers(peers, remote) => Try { (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => From d5618cb6d438a94fa0a17bf10f67898d9bd60070 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:24:36 +0300 Subject: [PATCH 111/119] added inv sending --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 9 ++++++++- src/main/scala/encry/nvg/NodeViewNMProcessor.scala | 13 +++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index a1585f5e7f..f82d227e89 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,7 +8,13 @@ import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{ + BroadcastModifier, + NotifyNodeAboutModifier, + RequestFromLocal, + ResponseFromLocal, + SendSyncInfo +} import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -104,6 +110,7 @@ class IntermediaryNVH( case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg case msg @ SendSyncInfo(_) => intermediaryNetwork ! msg + case msg @ NotifyNodeAboutModifier(_, _, _) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 693cab0618..824dee4acc 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -5,7 +5,13 @@ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{ + BroadcastModifier, + NotifyNodeAboutModifier, + RequestFromLocal, + ResponseFromLocal, + SendSyncInfo +} import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -63,12 +69,15 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case SemanticallySuccessfulModifier(_) => case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => + val extension: Seq[ModifierId] = + historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"\n\nComparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + - s"Comparison result is $comparison.\n " + s"Comparison result is $comparison. Extension length is: ${extension.size}.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) + context.parent ! NotifyNodeAboutModifier(remote, Header.modifierTypeId, extension.toList) case DataFromPeer(InvNetworkMessage(invData: InvData), remote) => if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) From 477c481f5bb88ef3ee656507e1ba079a71028973 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:54:01 +0300 Subject: [PATCH 112/119] one more test on pk --- src/main/scala/encry/network/PK.scala | 13 ++++++++++++- .../encry/network/ConnectWithNewPeerTests.scala | 8 +++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 9d84c09143..fd0362a5e1 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -15,7 +15,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingSta import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} -import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} @@ -76,6 +76,17 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } + case OutgoingConnectionFailed(peer) => + logger.info(s"Connection failed for: $peer.") + outgoingConnections -= peer + awaitingHandshakeConnections -= peer + val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 + if (connectionAttempts >= networkSettings.maxNumberOfReConnections) { + logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") + //todo think about penalty for the less time than general ban + //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) + peersForConnection -= peer + } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) case OtherNodeSyncingStatus(remote, comparison) => connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index a76153a211..13251c3b4e 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -146,9 +146,13 @@ class ConnectWithNewPeerTests extends WordSpecLike "remove peer from available we can't connect to" in { val networkController: TestProbe = TestProbe() val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( - PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(new InetSocketAddress("1.1.1.1", 1234))), settings.blackList), networkController.ref ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers @@ -165,8 +169,6 @@ class ConnectWithNewPeerTests extends WordSpecLike networkController.send(peersKeeper, OutgoingConnectionFailed(nextPeer.peer)) peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() From e2b35118849f3af375fa796c48fe75317a998996 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 11:19:50 +0300 Subject: [PATCH 113/119] more tests --- .../network/ConnectWithNewPeerTests.scala | 208 ++++++++++-------- 1 file changed, 118 insertions(+), 90 deletions(-) diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 13251c3b4e..da28e377e8 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -170,96 +170,124 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false } -// "remove peer from available if it has been banned" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection -// -// val peerHandler = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) -// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) -// -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// } -// "filter peers from network message" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection -// -// val peerHandler = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) -// -// val newPeerHandler = TestProbe() -// val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) -// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) -// networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// -// val peer = new InetSocketAddress("172.16.28.98", 9023) -// val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) -// -// networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true -// } -// "handle successful connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } -// "handle stop connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } + "remove peer from available if it has been banned" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(new InetSocketAddress("1.1.1.1", 1234))), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) + + networkController.send(peersKeeper, BanPeer(connectedPeer.socketAddress, ExpiredNumberOfConnections)) + networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) + + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + } + "filter peers from network message" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) + + val newPeerHandler = TestProbe() + val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) + + networkController.send(peersKeeper, BanPeer(connectedPeer.socketAddress, ExpiredNumberOfConnections)) + networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) + networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + + val peer = new InetSocketAddress("172.16.28.98", 9023) + val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) + + networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer.socketAddress)) + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true + } + "handle successful connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(peerAddr, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(peerAddr), System.currentTimeMillis())) + + networkController.send(peersKeeper, RequestPeerForConnection) + networkController.expectMsg(PeerForConnection(peerAddr)) + peersKeeper.underlyingActor.outgoingConnections.contains(peerAddr) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(peerAddr) shouldBe true + + networkController.send(peersKeeper, NewConnection(peerAddr, peerHandler.ref)) + networkController.expectMsg( + ConnectionVerified(peerAddr, peerHandler.ref, Outgoing)) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.connectedPeers.contains(peerAddr) shouldBe true + } + "handle stop connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val peersSenderProbe = TestProbe() + + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + val connectedPeer: ConnectedPeer = ConnectedPeer(msg.peer, peersSenderProbe.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(msg.peer), System.currentTimeMillis())) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(msg.peer, peersSenderProbe.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, peersSenderProbe.ref, Outgoing)) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe true + + peersKeeper ! ConnectionStopped(msg.peer) + peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + } // "handle failed connection process" in { // val networkController = TestProbe() // val peersSenderProbe = TestProbe() From 0e04d0a1599870a063ee18484fcc88e994664e42 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 16:51:13 +0300 Subject: [PATCH 114/119] Applying modifiers pipeline test added --- src/main/resources/application.conf | 2 + .../encry/settings/EncryAppSettings.scala | 3 +- .../encry/settings/NodeSettingsReader.scala | 3 +- .../scala/encry/view/history/History.scala | 11 +- .../history/HistoryModifiersValidator.scala | 2 +- .../network/ConnectWithNewPeerTests.scala | 2 +- src/test/scala/encry/nvg/PipelinesTests.scala | 305 +++++++++++++++++- .../scala/encry/utils/EncryGenerator.scala | 22 +- 8 files changed, 337 insertions(+), 13 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index d89602ab55..26e889813b 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -18,6 +18,8 @@ encry { miningDelay = 5s # Is CLI available useCli = true + #Debug regime + isTestMod = false } mempool { # The time during which the transaction is considered valid diff --git a/src/main/scala/encry/settings/EncryAppSettings.scala b/src/main/scala/encry/settings/EncryAppSettings.scala index b9f052c1c4..5fd4d90442 100644 --- a/src/main/scala/encry/settings/EncryAppSettings.scala +++ b/src/main/scala/encry/settings/EncryAppSettings.scala @@ -164,4 +164,5 @@ final case class NodeSettings(blocksToKeep: Int, numberOfMiningWorkers: Int, miningDelay: FiniteDuration, offlineGeneration: Boolean, - useCli: Boolean) + useCli: Boolean, + isTestMod: Boolean) diff --git a/src/main/scala/encry/settings/NodeSettingsReader.scala b/src/main/scala/encry/settings/NodeSettingsReader.scala index 208c7ded16..9cad2bb564 100644 --- a/src/main/scala/encry/settings/NodeSettingsReader.scala +++ b/src/main/scala/encry/settings/NodeSettingsReader.scala @@ -14,6 +14,7 @@ trait NodeSettingsReader { cfg.as[Int](s"$path.numberOfMiningWorkers"), cfg.as[FiniteDuration](s"$path.miningDelay"), cfg.as[Boolean](s"$path.offlineGeneration"), - cfg.as[Boolean](s"$path.useCli") + cfg.as[Boolean](s"$path.useCli"), + cfg.as[Boolean](s"$path.isTestMod") ) } \ No newline at end of file diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index d2a6154eb3..6ae85f3887 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -1,6 +1,8 @@ package encry.view.history import java.io.File + +import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ @@ -10,14 +12,12 @@ import encry.storage.iodb.versionalIODB.IODBHistoryWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.NetworkTimeProvider import encry.view.history.storage.HistoryStorage -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options -import cats.syntax.either._ -import supertagged.@@ /** * History implementation. It is processing persistent modifiers generated locally or received from the network. @@ -177,17 +177,16 @@ object History extends StrictLogging { if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) new History with HistoryHeadersProcessor with FastSyncProcessor { override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } else new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } - } } \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 0741251609..d777887796 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -71,7 +71,7 @@ trait HistoryModifiersValidator extends HistoryApi { _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(powSchemeValidationResult.isRight, (), + _ <- Either.cond(if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index a76153a211..3d56eb7d7d 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -166,7 +166,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true + //peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 05d227e223..2feb7a6f77 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -1,5 +1,308 @@ package encry.nvg -class PipelinesTests { +import java.net.InetSocketAddress +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } +import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp +import encry.consensus.EncrySupplyController +import encry.modifiers.InstanceFactory +import encry.modifiers.mempool.TransactionFactory +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.utils.implicits.UTXO.{ combineAll, _ } +import encry.utils.{ FileHelper, Mnemonic, NetworkTimeProvider, TestHelper } +import encry.view.history.History +import encry.view.state.UtxoState +import encry.view.state.avlTree.utils.implicits.Instances._ +import encry.view.wallet.AccountManager +import org.encryfoundation.common.crypto.PrivateKey25519 +import org.encryfoundation.common.crypto.equihash.EquihashSolution +import org.encryfoundation.common.modifiers.history._ +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.state.box.AssetBox +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height } +import org.encryfoundation.common.utils.constants.TestNetConstants +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } + +import scala.concurrent.duration._ + +class PipelinesTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with StrictLogging { + + override def afterAll(): Unit = system.terminate() + + "Node view pipelines" should { + "correct process modifier from the network async" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val intermediaryParent = TestProbe() + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary: ActorRef = intermediaryParent.childActorOf( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + logger.info(s"Sent to nvh actor header ${block.encodedId}") + } + + Thread.sleep(8000) + + networkIntermediary.expectMsgPF(15.seconds) { + case SemanticallySuccessfulModifier(mod) => + blocks.exists(_.id.sameElements(mod.id)) shouldBe true + case msg @ SendSyncInfo(_) => + case msg @ BroadcastModifier(modifierTypeId, modifierId) => + blocks.exists(_.id.sameElements(modifierId)) shouldBe true + case RegisterMessagesHandler(_, _) => + case RegisterForModsHandling => + case RequestFromLocal(s, m, mods) => + mods.size shouldBe blocks.size + mods.zip(blocks).forall { + case (id, block) => id.sameElements(block.id) + } shouldBe true + } + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(10000) + + networkIntermediary.expectMsgPF(15.seconds) { + case SemanticallySuccessfulModifier(mod) => + blocks.exists(_.id.sameElements(mod.id)) shouldBe true + case msg @ SendSyncInfo(_) => + case RegisterForModsHandling => + case msg @ BroadcastModifier(modifierTypeId, modifierId) => + blocks.exists(_.id.sameElements(modifierId)) shouldBe true + case RegisterMessagesHandler(_, _) => + case RequestFromLocal(s, m, mods) => + } + + } + "correct process modifier from the network sync" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary = TestActorRef[IntermediaryNVH]( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + logger.info(s"Sent to nvh actor header ${block.encodedId}") + } + + Thread.sleep(8000) + + intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 + intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe -1 + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(10000) + + intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 + intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 + + } + } + +} + +object PipelinesTests extends InstanceFactory { + + val key: PrivateKey25519 = TestHelper.genKeys(1).head + + def generateValidForHistoryAndStateBlocks(blocksQty: Int): (History, UtxoState, List[Block]) = { + (0 to blocksQty).foldLeft(generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + List.empty[Block]) { + case ((history, state, blocks), i) => + val blockNext: Block = + if (i > 0) { + val boxes: Seq[AssetBox] = + history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { + case a: AssetBox if a.amount > 13 => a + } + val txs: Vector[Transaction] = + generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) + val feesTotal = txs.map(_.fee).sum + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) + val resTxs = txs :+ coinbase + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + TestNetConstants.Version, + history.getBestHeaderId.get, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } else { + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) + val resTxs = List(coinbase) + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + 0: Byte, + Header.GenesisParentId, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } + val h = history + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val s = state + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + (h, s, blocks :+ blockNext) + } + } } diff --git a/src/test/scala/encry/utils/EncryGenerator.scala b/src/test/scala/encry/utils/EncryGenerator.scala index 05879b45cb..87e1bf442a 100644 --- a/src/test/scala/encry/utils/EncryGenerator.scala +++ b/src/test/scala/encry/utils/EncryGenerator.scala @@ -159,6 +159,24 @@ trait EncryGenerator extends Settings { ) } + def generatePaymentTransactions(privKey: PrivateKey25519, + boxes: IndexedSeq[AssetBox], + numberOfInputs: Int, + numberOfOutputs: Int): Vector[Transaction] = + (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { + case ((boxesLocal, transactions), _) => + val tx: Transaction = defaultPaymentTransactionScratch( + privKey, + fee = 11, + timestamp = 11L, + useBoxes = boxesLocal.take(numberOfInputs), + recipient = randomAddress, + amount = 1, + numOfOutputs = numberOfOutputs + ) + (boxesLocal.drop(numberOfInputs), transactions :+ tx) + }._2 + def generatePaymentTransactions(boxes: IndexedSeq[AssetBox], numberOfInputs: Int, numberOfOutputs: Int): Vector[Transaction] = @@ -166,11 +184,11 @@ trait EncryGenerator extends Settings { case ((boxesLocal, transactions), _) => val tx: Transaction = defaultPaymentTransactionScratch( privKey, - fee = 111, + fee = 0, timestamp = 11L, useBoxes = boxesLocal.take(numberOfInputs), recipient = randomAddress, - amount = 10000, + amount = 1, numOfOutputs = numberOfOutputs ) (boxesLocal.drop(numberOfInputs), transactions :+ tx) From 93303834bc1fef7f426ba72146c02975459c03bb Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 16:51:51 +0300 Subject: [PATCH 115/119] fixed modifiers propagation --- src/main/scala/encry/EncryApp.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/EncryApp.scala b/src/main/scala/encry/EncryApp.scala index db1f23ead8..a1d66eed3e 100644 --- a/src/main/scala/encry/EncryApp.scala +++ b/src/main/scala/encry/EncryApp.scala @@ -4,14 +4,14 @@ import java.io.File import java.net.InetAddress import java.nio.file.Files import akka.actor.SupervisorStrategy.Restart -import akka.actor.{ActorRef, ActorSystem, OneForOneStrategy, Props} +import akka.actor.{ ActorRef, ActorSystem, OneForOneStrategy, Props } import akka.http.scaladsl.Http import akka.http.scaladsl.model.HttpResponse import akka.http.scaladsl.server.ExceptionHandler import akka.stream.ActorMaterializer import com.typesafe.scalalogging.StrictLogging import encry.api.http.routes._ -import encry.api.http.{ApiRoute, CompositeHttpService} +import encry.api.http.{ ApiRoute, CompositeHttpService } import encry.settings.EncryAppSettings import encry.stats.Zombie import encry.utils.NetworkTimeProvider @@ -20,7 +20,7 @@ import kamon.influxdb.InfluxDBReporter import kamon.system.SystemMetrics import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContextExecutor, Future} +import scala.concurrent.{ Await, ExecutionContextExecutor, Future } import scala.io.Source import scala.language.postfixOps From 5adef1944c900c92350317e42f67ab3aafbd8acd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 20:09:00 +0300 Subject: [PATCH 116/119] added test for nvg forks --- .../encry/view/history/HistoryReader.scala | 9 +- src/test/scala/encry/nvg/PipelinesTests.scala | 266 +++++++++++++++++- 2 files changed, 262 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 7d781b3f00..233ee8f491 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -2,7 +2,6 @@ package encry.view.history import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import encry.modifiers.history.HeaderChain -import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -44,6 +43,10 @@ trait HistoryReader { def syncInfo: SyncInfo def isFastSyncInProcess: Boolean + + def getBestHeader: Option[Header] + + def getBestBlock: Option[Block] } object HistoryReader { @@ -64,6 +67,8 @@ object HistoryReader { def getBlockByHeader(header: Header): Option[Block] = None def headerIdsAtHeight(height: Int): List[ModifierId] = List.empty[ModifierId] def lastHeaders(count: Int): HeaderChain = HeaderChain.empty + def getBestHeader: Option[Header] = None + def getBestBlock: Option[Block] = None } def apply(history: History): HistoryReader = new HistoryReader { @@ -83,5 +88,7 @@ object HistoryReader { def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) + def getBestHeader: Option[Header] = history.getBestHeader + def getBestBlock: Option[Block] = history.getBestBlock } } diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 2feb7a6f77..11691b812a 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -71,7 +71,14 @@ class PipelinesTests val remote = new InetSocketAddress("0.0.0.0", 9001) - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + )) blocks.reverse.foreach { block => intermediary ! ModifierFromNetwork( @@ -151,7 +158,14 @@ class PipelinesTests val remote = new InetSocketAddress("0.0.0.0", 9001) - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + )) blocks.reverse.foreach { block => intermediary ! ModifierFromNetwork( @@ -183,23 +197,103 @@ class PipelinesTests intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 } + "work with forks correctly" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary = TestActorRef[IntermediaryNVH]( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val ((h1, s1, b1), (h2, s2, b2)) = PipelinesTests.genForOn(5) + + b1.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.header.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + } + + Thread.sleep(3000) + + b1.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(5000) + + b2.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.header.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + } + + Thread.sleep(3000) + + b2.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(5000) + + intermediary.underlyingActor.historyReader.getBestBlock.get.id.sameElements(b2.last.id) shouldBe true + intermediary.underlyingActor.historyReader.getBestHeader.get.id.sameElements(b2.last.id) shouldBe true + + } } } -object PipelinesTests extends InstanceFactory { +object PipelinesTests extends InstanceFactory with StrictLogging { val key: PrivateKey25519 = TestHelper.genKeys(1).head - def generateValidForHistoryAndStateBlocks(blocksQty: Int): (History, UtxoState, List[Block]) = { - (0 to blocksQty).foldLeft(generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - ), - List.empty[Block]) { + def generateValidForHistoryAndStateBlocks( + blocksQty: Int, + history: History, + utxo: UtxoState, + from: Int = 0 + ): (History, UtxoState, List[Block]) = { + (from to from + blocksQty).foldLeft( + history, + utxo, + List.empty[Block] + ) { case ((history, state, blocks), i) => val blockNext: Block = if (i > 0) { @@ -305,4 +399,152 @@ object PipelinesTests extends InstanceFactory { (h, s, blocks :+ blockNext) } } + + def genForOn( + blocksQty: Int + ): ((History, UtxoState, List[Block]), (History, UtxoState, List[Block])) = { + val (h, s, h1, s1, b) = (0 to blocksQty).foldLeft( + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + List.empty[Block] + ) { + case ((history, state, h1, s1, blocks), i) => + val blockNext: Block = + if (i > 0) { + val boxes: Seq[AssetBox] = + history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { + case a: AssetBox if a.amount > 13 => a + } + val txs: Vector[Transaction] = + generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) + val feesTotal = txs.map(_.fee).sum + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) + val resTxs = txs :+ coinbase + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + TestNetConstants.Version, + history.getBestHeaderId.get, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } else { + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) + val resTxs = List(coinbase) + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + 0: Byte, + Header.GenesisParentId, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } + val h = history + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val s = state + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + + val his1 = h1 + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val st1 = s1 + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + (h, s, his1, st1, blocks :+ blockNext) + } + val (h11, s11, b11) = generateValidForHistoryAndStateBlocks(5, h, s, h.getBestBlockHeight + 1) + val (h22, s22, b22) = generateValidForHistoryAndStateBlocks(10, h1, s1, h1.getBestBlockHeight + 1) + ((h11, s11, b11 ++ b), (h22, s22, b22)) + } } From 64c9e1b415b8577514539e05f67c763620f322b3 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 10:24:40 +0300 Subject: [PATCH 117/119] miner pipeline --- .../encry/view/history/HistoryModifiersValidator.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index d777887796..3a80a90dde 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -64,14 +64,14 @@ trait HistoryModifiersValidator extends HistoryApi { s" not greater by 1 than parent's ${parent.height}")) _ <- Either.cond(!historyStorage.containsMod(h.id), (), HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) - _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, (), + _ <- Either.cond( if (settings.node.isTestMod) true else realDifficulty(h) >= h.requiredDifficulty, (), HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) - _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), + _ <- Either.cond( if (settings.node.isTestMod) true else requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), + _ <- Either.cond( if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), From 155d83bc73692ddaef4e6a37d789e169666d11fe Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 11:01:06 +0300 Subject: [PATCH 118/119] recover some tests, fix dm logic with txs --- src/main/scala/encry/network/DM.scala | 16 +- .../network/ConnectWithNewPeerTests.scala | 203 ++++++++++-------- .../DeliveryManagerTests/DMUtils.scala | 7 +- ...DeliveryManagerReRequestModifiesSpec.scala | 52 ++--- .../DeliveryManagerRequestModifiesSpec.scala | 184 ++++++++-------- 5 files changed, 237 insertions(+), 225 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index dbd2a2fc1f..2bd849cdae 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -15,6 +15,7 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import scala.collection.mutable import scala.concurrent.Future @@ -40,18 +41,19 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifiers += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) => + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId)))=> expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case RequestSent(_, _, _) => //do nothing - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts < networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> - context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) - logger.info(s"Re-request modifier ${Algos.encode(modId)}") - context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! - AwaitingRequest(peer, modTypeId, modId, attempts + 1) - ) + case AwaitingRequest(peer, modTypeId, modId, attempts) if + attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId)) && modTypeId != Transaction.modifierTypeId => + context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + logger.info(s"Re-request modifier ${Algos.encode(modId)}") + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, attempts + 1) + ) case AwaitingRequest(peer, _, modId, attempts) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index da28e377e8..a916791861 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -51,7 +51,6 @@ class ConnectWithNewPeerTests extends WordSpecLike val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers networkController.send(peersKeeper, RequestPeerForConnection) - println(peersKeeper.underlyingActor.knownPeers.map(PeerForConnection)) networkController.expectMsg(RegisterMessagesHandler(Seq( PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" @@ -288,95 +287,117 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe false peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true } -// "handle failed connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer doesn't contains in black list and connected peers collection" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer contain in black list" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(remoteAddress), System.currentTimeMillis())) -// -// peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer contain in connected peers" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(remoteAddress), System.currentTimeMillis())) -// -// peersKeeper ! HandshakedDone(connectedPeer) -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers true" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while peer is equal to local address" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection( -// new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle outgoing connection" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// peersKeeper ! RequestPeerForConnection -// peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true -// networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) -// } + "handle failed connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val peersSenderProbe = TestProbe() + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(msg.peer, peersSenderProbe.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, peersSenderProbe.ref, Outgoing)) + + peersKeeper ! OutgoingConnectionFailed(msg.peer) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + } + "handle incoming connections correctly while connection with only known peers false " + + "and incoming peer doesn't contains in black list and connected peers collection" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectMsg(ConnectionVerified(peerAddr, remoteConnectionTestProbe.ref, Incoming)) + peersKeeper.stop() + } + "handle incoming connections correctly while connection with only known peers false " + + "and incoming peer contain in black list" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val remoteConnectionTestProbe = TestProbe() + + peersKeeper ! BanPeer(peerAddr, SyntacticallyInvalidPersistentModifier) + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle incoming connections correctly while connection with only known peers true" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle incoming connections correctly while peer is equal to local address" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection( + new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle outgoing connection" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + val remoteConnectionTestProbe = TestProbe() + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + networkController.send(peersKeeper, NewConnection(msg.peer, remoteConnectionTestProbe.ref)) + networkController.expectMsg( + ConnectionVerified(msg.peer, remoteConnectionTestProbe.ref, Outgoing)) + } } } \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 0f62964c0d..b0d124080b 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -24,14 +24,15 @@ object DMUtils extends InstanceFactory { def initialiseDeliveryManager(isBlockChainSynced: Boolean, isMining: Boolean, settings: EncryAppSettings) - (implicit actorSystem: ActorSystem): (TestActorRef[DM], History) = { + (implicit actorSystem: ActorSystem): (TestProbe, TestActorRef[DM], History) = { val history: History = generateDummyHistory(settings) - val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network)) + val networkRouter = TestProbe() + val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network), networkRouter.ref) deliveryManager ! UpdatedHistory(history) if (isMining) deliveryManager ! StartMining else deliveryManager ! DisableMining if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced - (deliveryManager, history) + (networkRouter, deliveryManager, history) } def generateBlocks(qty: Int, history: History): (History, List[Block]) = diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index e29d0daafd..d3fe361312 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -10,7 +10,8 @@ import encry.modifiers.InstanceFactory import encry.network.DM.RequestSent import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} import encry.network.PeersKeeper.UpdatedPeersCollection @@ -21,7 +22,7 @@ import encry.settings.TestNetSettings import encry.view.history.History import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} @@ -39,16 +40,16 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike override def afterAll(): Unit = system.terminate() - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestProbe, TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { - val (deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (networkRouter, deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) + (networkRouter, deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) } "ReRequestModifies" should { @@ -84,28 +85,25 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike // deliveryManager.stop() // } "not re-ask unnecessary modifiers" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - + val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) val address1 = new InetSocketAddress("123.123.123.123", 9001) val handler1: TestProbe = TestProbe() val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, Handshake(protocolToBytes(testNetSettings.network.appVersion), "123.123.123.123", Some(address1), System.currentTimeMillis())) - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - val header: ModifierId = headersIds.head deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) //await one re-ask - handler1.expectMsgAllOf( + networkRouter.expectMsgAllOf( testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) + RequestFromLocal(Some(address1), Header.modifierTypeId, List(header)), + RequestFromLocal(Some(address1), Header.modifierTypeId, List(header)) ) val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray @@ -115,7 +113,7 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike deliveryManager.stop() } "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { - val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, headerIds.head) //this thread sleep is using for expecting modifier removal @@ -124,27 +122,17 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike deliveryManager.stop() } "not re-ask transactions" in { - val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() - + val (networkRouter, deliveryManager, _, _, _, _, _, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) - transactions.foreach(txId => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, txId)) + transactions.foreach(txId => deliveryManager ! RequestSent(address1, Transaction.modifierTypeId, txId)) - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) - ) - handler1.expectNoMsg(10.seconds) + networkRouter.expectNoMsg(testNetSettings.network.deliveryTimeout + 10.seconds) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) deliveryManager.stop() } diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 3e2d241362..4aa685b794 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -37,21 +37,21 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestProbe, TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (networkRouter, deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + (networkRouter, deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) } "RequestModifies" should { "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -61,7 +61,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -72,7 +72,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -86,7 +86,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -101,7 +101,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -114,89 +114,89 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte assert(deliveryManager.underlyingActor.expectedModifiers.size == blocks.size) deliveryManager.stop() } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - - - val header: Header = blocks.head.header - - deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) - - deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - - - val header: Header = blocks.head.header - - deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } + //todo: reinit +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) +// +// deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } + //todo: reinit +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val (_, deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) val txs: Seq[Transaction] = genInvalidPaymentTxs(1) val address1 = new InetSocketAddress("123.123.123.123", 9001) @@ -216,7 +216,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val (_, deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) val txs: Seq[Transaction] = genInvalidPaymentTxs(1) val address1 = new InetSocketAddress("123.123.123.123", 9001) @@ -236,7 +236,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + val (_, deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) val address1 = new InetSocketAddress("123.123.123.123", 9001) val handler1: TestProbe = TestProbe() From b40968df21976b49a0a43e2419a375a8e419a945 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 14:34:36 +0300 Subject: [PATCH 119/119] fix some tests --- src/main/scala/encry/network/DM.scala | 4 -- src/main/scala/encry/network/PK.scala | 2 +- .../scala/encry/network/BlackListTests.scala | 4 +- ...DeliveryManagerReRequestModifiesSpec.scala | 67 ++++++++----------- 4 files changed, 31 insertions(+), 46 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2bd849cdae..fa4944da65 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,9 +1,7 @@ package encry.network import java.net.InetSocketAddress - import akka.actor.{Actor, Props} -import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent, RequestStatus} import encry.network.Messages.MessageToNetwork.RequestFromLocal @@ -16,9 +14,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction - import scala.collection.mutable -import scala.concurrent.Future case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index fd0362a5e1..a5375ec127 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,7 +2,7 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index ada46f14a7..30ea0749ef 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -53,8 +53,8 @@ class BlackListTests extends WordSpecLike val newBL1 = newBL.cleanupBlackList newBL1.contains(peer) shouldBe false } - "don't remove peer from black list before ban time expired" in { - val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + "not remove peer from black list before ban time expired" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 minute)) val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress val newBL = blackList.banPeer(SentInvForPayload, peer) val newBL1 = newBL.cleanupBlackList diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index d3fe361312..112f3a28b0 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -4,30 +4,23 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.Older import encry.modifiers.InstanceFactory +import encry.network.DM import encry.network.DM.RequestSent -import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManagerTests.DMUtils._ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.TestNetSettings import encry.view.history.History import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scala.concurrent.duration._ import scala.collection.mutable.WrappedArray +import scala.concurrent.duration._ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll @@ -55,35 +48,31 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike "ReRequestModifies" should { "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + //todo: move to message builder tests -// "expectedModifiers collection after all attempts will expire" in { -// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: ModifierId = headersIds.head -// -// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) -// handler1.expectMsgAllOf( -// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) -// ) -// //this thread sleep is using for expecting modifier removal -// Thread.sleep(6000) -// -// assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) -// deliveryManager.stop() -// } + "expectedModifiers collection after all attempts will expire" in { + val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val header: ModifierId = headersIds.head + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) + networkRouter.expectMsgAllOf( + testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), + RequestFromLocal(Some(cp1.socketAddress), Header.modifierTypeId, List(header)), + RequestFromLocal(Some(cp1.socketAddress), Header.modifierTypeId, List(header)), + ) + //this thread sleep is using for expecting modifier removal + Thread.sleep(6000) + + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } "not re-ask unnecessary modifiers" in { val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() networkRouter.expectMsg(RegisterMessagesHandler(Seq(