From 64987d741e0363d689d12e6be0c3f85202cd0d5b Mon Sep 17 00:00:00 2001 From: John Long Date: Fri, 27 Feb 2026 09:41:59 -0800 Subject: [PATCH 01/31] Make SafetyWindow a Parameter (#809) # Make SafetyWindow a Parameter ## What problem does this solve? During a recent chain halt, we skipped making SafetyWindow a parameter to reduce risk and expedite recovery. We fixed it at 50. The problem: - Most urgently, this broke our integration tests, because there was no way to override the value, breaking all CPoC Integration tests because Epochs are shorter than 50 blocks for testing, meaning no CPoC ever executes - Secondarily, but also important, this puts an important value impacting chain execution under proper governance control. ## How do you know this is a real problem? Currently there are at least 4 failing integration tests, obscuring our ability to verify other code problems. ## How does this solve the problem? - Add a new parameter to EpochParameters for SafetyWindow (default to 50, per the current value) - Remove the hard coded value - Fix tests to override this to 0 ## What risks does this introduce? How can we mitigate those risks? - There is very little risk, but an upgrade issue would be caught via testnet update testing, done for every upgrade. ## How do you know this PR fixes the problem? Integration Tests now pass when they failed. --- .../api/inference/inference/params.pulsar.go | 867 ++++++++++-------- inference-chain/app/upgrades.go | 2 + .../app/upgrades/v0_2_11/constants.go | 3 + .../app/upgrades/v0_2_11/upgrades.go | 58 ++ .../app/upgrades/v0_2_11/upgrades_test.go | 11 + .../proto/inference/inference/params.proto | 1 + inference-chain/test_genesis_overrides.json | 3 +- .../x/inference/module/confirmation_poc.go | 4 +- inference-chain/x/inference/types/params.go | 4 + .../x/inference/types/params.pb.go | 473 +++++----- test-net-cloud/nebius/genesis-overrides.json | 1 + testermint/src/main/kotlin/data/AppExport.kt | 1 + .../kotlin/ConfirmationPoCMultiNodeTests.kt | 1 + 13 files changed, 805 insertions(+), 624 deletions(-) create mode 100644 inference-chain/app/upgrades/v0_2_11/constants.go create mode 100644 inference-chain/app/upgrades/v0_2_11/upgrades.go create mode 100644 inference-chain/app/upgrades/v0_2_11/upgrades_test.go diff --git a/inference-chain/api/inference/inference/params.pulsar.go b/inference-chain/api/inference/inference/params.pulsar.go index dbc644811..828977fa6 100644 --- a/inference-chain/api/inference/inference/params.pulsar.go +++ b/inference-chain/api/inference/inference/params.pulsar.go @@ -3564,6 +3564,7 @@ var ( fd_EpochParams_inference_pruning_max protoreflect.FieldDescriptor fd_EpochParams_poc_pruning_max protoreflect.FieldDescriptor fd_EpochParams_poc_slot_allocation protoreflect.FieldDescriptor + fd_EpochParams_confirmation_poc_safety_window protoreflect.FieldDescriptor ) func init() { @@ -3583,6 +3584,7 @@ func init() { fd_EpochParams_inference_pruning_max = md_EpochParams.Fields().ByName("inference_pruning_max") fd_EpochParams_poc_pruning_max = md_EpochParams.Fields().ByName("poc_pruning_max") fd_EpochParams_poc_slot_allocation = md_EpochParams.Fields().ByName("poc_slot_allocation") + fd_EpochParams_confirmation_poc_safety_window = md_EpochParams.Fields().ByName("confirmation_poc_safety_window") } var _ protoreflect.Message = (*fastReflection_EpochParams)(nil) @@ -3734,6 +3736,12 @@ func (x *fastReflection_EpochParams) Range(f func(protoreflect.FieldDescriptor, return } } + if x.ConfirmationPocSafetyWindow != int64(0) { + value := protoreflect.ValueOfInt64(x.ConfirmationPocSafetyWindow) + if !f(fd_EpochParams_confirmation_poc_safety_window, value) { + return + } + } } // Has reports whether a field is populated. @@ -3777,6 +3785,8 @@ func (x *fastReflection_EpochParams) Has(fd protoreflect.FieldDescriptor) bool { return x.PocPruningMax != int64(0) case "inference.inference.EpochParams.poc_slot_allocation": return x.PocSlotAllocation != nil + case "inference.inference.EpochParams.confirmation_poc_safety_window": + return x.ConfirmationPocSafetyWindow != int64(0) default: if fd.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -3821,6 +3831,8 @@ func (x *fastReflection_EpochParams) Clear(fd protoreflect.FieldDescriptor) { x.PocPruningMax = int64(0) case "inference.inference.EpochParams.poc_slot_allocation": x.PocSlotAllocation = nil + case "inference.inference.EpochParams.confirmation_poc_safety_window": + x.ConfirmationPocSafetyWindow = int64(0) default: if fd.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -3879,6 +3891,9 @@ func (x *fastReflection_EpochParams) Get(descriptor protoreflect.FieldDescriptor case "inference.inference.EpochParams.poc_slot_allocation": value := x.PocSlotAllocation return protoreflect.ValueOfMessage(value.ProtoReflect()) + case "inference.inference.EpochParams.confirmation_poc_safety_window": + value := x.ConfirmationPocSafetyWindow + return protoreflect.ValueOfInt64(value) default: if descriptor.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -3927,6 +3942,8 @@ func (x *fastReflection_EpochParams) Set(fd protoreflect.FieldDescriptor, value x.PocPruningMax = value.Int() case "inference.inference.EpochParams.poc_slot_allocation": x.PocSlotAllocation = value.Message().Interface().(*Decimal) + case "inference.inference.EpochParams.confirmation_poc_safety_window": + x.ConfirmationPocSafetyWindow = value.Int() default: if fd.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -3978,6 +3995,8 @@ func (x *fastReflection_EpochParams) Mutable(fd protoreflect.FieldDescriptor) pr panic(fmt.Errorf("field inference_pruning_max of message inference.inference.EpochParams is not mutable")) case "inference.inference.EpochParams.poc_pruning_max": panic(fmt.Errorf("field poc_pruning_max of message inference.inference.EpochParams is not mutable")) + case "inference.inference.EpochParams.confirmation_poc_safety_window": + panic(fmt.Errorf("field confirmation_poc_safety_window of message inference.inference.EpochParams is not mutable")) default: if fd.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -4020,6 +4039,8 @@ func (x *fastReflection_EpochParams) NewField(fd protoreflect.FieldDescriptor) p case "inference.inference.EpochParams.poc_slot_allocation": m := new(Decimal) return protoreflect.ValueOfMessage(m.ProtoReflect()) + case "inference.inference.EpochParams.confirmation_poc_safety_window": + return protoreflect.ValueOfInt64(int64(0)) default: if fd.IsExtension() { panic(fmt.Errorf("proto3 declared messages do not support extensions: inference.inference.EpochParams")) @@ -4132,6 +4153,9 @@ func (x *fastReflection_EpochParams) ProtoMethods() *protoiface.Methods { l = options.Size(x.PocSlotAllocation) n += 1 + l + runtime.Sov(uint64(l)) } + if x.ConfirmationPocSafetyWindow != 0 { + n += 1 + runtime.Sov(uint64(x.ConfirmationPocSafetyWindow)) + } if x.unknownFields != nil { n += len(x.unknownFields) } @@ -4161,6 +4185,11 @@ func (x *fastReflection_EpochParams) ProtoMethods() *protoiface.Methods { i -= len(x.unknownFields) copy(dAtA[i:], x.unknownFields) } + if x.ConfirmationPocSafetyWindow != 0 { + i = runtime.EncodeVarint(dAtA, i, uint64(x.ConfirmationPocSafetyWindow)) + i-- + dAtA[i] = 0x78 + } if x.PocSlotAllocation != nil { encoded, err := options.Marshal(x.PocSlotAllocation) if err != nil { @@ -4572,6 +4601,25 @@ func (x *fastReflection_EpochParams) ProtoMethods() *protoiface.Methods { return protoiface.UnmarshalOutput{NoUnkeyedLiterals: input.NoUnkeyedLiterals, Flags: input.Flags}, err } iNdEx = postIndex + case 15: + if wireType != 0 { + return protoiface.UnmarshalOutput{NoUnkeyedLiterals: input.NoUnkeyedLiterals, Flags: input.Flags}, fmt.Errorf("proto: wrong wireType = %d for field ConfirmationPocSafetyWindow", wireType) + } + x.ConfirmationPocSafetyWindow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protoiface.UnmarshalOutput{NoUnkeyedLiterals: input.NoUnkeyedLiterals, Flags: input.Flags}, runtime.ErrIntOverflow + } + if iNdEx >= l { + return protoiface.UnmarshalOutput{NoUnkeyedLiterals: input.NoUnkeyedLiterals, Flags: input.Flags}, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + x.ConfirmationPocSafetyWindow |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := runtime.Skip(dAtA[iNdEx:]) @@ -16255,6 +16303,7 @@ type EpochParams struct { InferencePruningMax int64 `protobuf:"varint,12,opt,name=inference_pruning_max,json=inferencePruningMax,proto3" json:"inference_pruning_max,omitempty"` PocPruningMax int64 `protobuf:"varint,13,opt,name=poc_pruning_max,json=pocPruningMax,proto3" json:"poc_pruning_max,omitempty"` PocSlotAllocation *Decimal `protobuf:"bytes,14,opt,name=poc_slot_allocation,json=pocSlotAllocation,proto3" json:"poc_slot_allocation,omitempty"` // Fraction of slots allocated to PoC (0.0 to 1.0, default 0.5) + ConfirmationPocSafetyWindow int64 `protobuf:"varint,15,opt,name=confirmation_poc_safety_window,json=confirmationPocSafetyWindow,proto3" json:"confirmation_poc_safety_window,omitempty"` } func (x *EpochParams) Reset() { @@ -16376,6 +16425,13 @@ func (x *EpochParams) GetPocSlotAllocation() *Decimal { return nil } +func (x *EpochParams) GetConfirmationPocSafetyWindow() int64 { + if x != nil { + return x.ConfirmationPocSafetyWindow + } + return 0 +} + type ValidationParams struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -17775,7 +17831,7 @@ var file_inference_inference_params_proto_rawDesc = []byte{ 0x18, 0x74, 0x6f, 0x70, 0x5f, 0x6d, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x15, 0x74, 0x6f, 0x70, 0x4d, 0x69, 0x6e, 0x65, 0x72, 0x56, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, - 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x82, 0x06, 0x0a, + 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xc7, 0x06, 0x0a, 0x0b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, @@ -17823,425 +17879,430 @@ var file_inference_inference_params_proto_rawDesc = []byte{ 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x11, 0x70, 0x6f, 0x63, 0x53, 0x6c, 0x6f, - 0x74, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, - 0x01, 0x22, 0x8e, 0x0e, 0x0a, 0x10, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x4c, 0x0a, 0x13, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x5f, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, - 0x6c, 0x52, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x61, 0x6d, 0x70, - 0x5f, 0x75, 0x70, 0x5f, 0x6d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d, 0x69, 0x6e, 0x52, 0x61, 0x6d, 0x70, 0x55, - 0x70, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3b, 0x0a, - 0x0a, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, - 0x09, 0x70, 0x61, 0x73, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x52, 0x0a, 0x16, 0x6d, 0x69, - 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x61, 0x76, 0x65, - 0x72, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, - 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x12, 0x52, - 0x0a, 0x16, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x6d, 0x61, - 0x78, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x76, 0x65, 0x72, 0x61, - 0x67, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x65, - 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x12, - 0x22, 0x0a, 0x0d, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x73, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x78, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x73, 0x54, 0x6f, - 0x4d, 0x61, 0x78, 0x12, 0x43, 0x0a, 0x1e, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x63, - 0x75, 0x74, 0x6f, 0x66, 0x66, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1b, 0x66, 0x75, 0x6c, - 0x6c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x72, 0x61, 0x66, 0x66, - 0x69, 0x63, 0x43, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x12, 0x52, 0x0a, 0x16, 0x6d, 0x69, 0x6e, 0x5f, - 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x61, 0x6c, 0x66, 0x77, - 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, + 0x74, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a, 0x1e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x6f, 0x63, 0x5f, + 0x73, 0x61, 0x66, 0x65, 0x74, 0x79, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x6f, 0x63, 0x53, 0x61, 0x66, 0x65, 0x74, 0x79, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, + 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x8e, 0x0e, 0x0a, 0x10, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x4c, 0x0a, 0x13, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x72, 0x61, + 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, - 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x6c, 0x66, 0x77, 0x61, 0x79, 0x12, 0x41, 0x0a, 0x1d, - 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x63, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x18, 0x0a, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x1a, 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x43, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x12, - 0x52, 0x0a, 0x16, 0x6d, 0x69, 0x73, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, - 0x67, 0x65, 0x5f, 0x63, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x11, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x50, 0x6f, 0x73, + 0x69, 0x74, 0x69, 0x76, 0x65, 0x52, 0x61, 0x74, 0x65, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x69, 0x6e, + 0x5f, 0x72, 0x61, 0x6d, 0x70, 0x5f, 0x75, 0x70, 0x5f, 0x6d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d, 0x69, 0x6e, + 0x52, 0x61, 0x6d, 0x70, 0x55, 0x70, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, + 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x09, 0x70, 0x61, 0x73, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x52, 0x0a, 0x16, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x6d, - 0x69, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x43, 0x75, 0x74, - 0x6f, 0x66, 0x66, 0x12, 0x50, 0x0a, 0x15, 0x6d, 0x69, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, - 0x52, 0x13, 0x6d, 0x69, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x50, 0x65, - 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x12, 0x31, 0x0a, 0x14, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x45, 0x78, - 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x61, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x10, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x41, 0x64, - 0x76, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x1d, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, - 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x5f, 0x6b, 0x62, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x65, 0x73, - 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x50, 0x65, 0x72, - 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4b, 0x62, 0x12, 0x5c, 0x0a, 0x1b, 0x69, 0x6e, 0x76, 0x61, 0x6c, - 0x69, 0x64, 0x5f, 0x72, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x19, 0x69, 0x6e, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x52, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x12, 0x67, 0x0a, 0x21, 0x62, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x1e, - 0x62, 0x61, 0x64, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x49, 0x6e, - 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x74, 0x65, 0x12, 0x56, - 0x0a, 0x18, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68, - 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x16, - 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x54, 0x68, 0x72, - 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x56, 0x0a, 0x18, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, - 0x6d, 0x65, 0x5f, 0x67, 0x6f, 0x6f, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, - 0x67, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, - 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x16, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x47, 0x6f, 0x6f, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, 0x54, - 0x0a, 0x17, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x62, 0x61, 0x64, 0x5f, 0x70, - 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x15, 0x64, - 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x61, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, - 0x74, 0x61, 0x67, 0x65, 0x12, 0x4e, 0x0a, 0x14, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x5f, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x15, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, - 0x52, 0x12, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x48, 0x54, 0x68, 0x72, 0x65, 0x73, - 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x5e, 0x0a, 0x1c, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x5f, 0x72, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x1a, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, - 0x6d, 0x65, 0x52, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x12, 0x54, 0x0a, 0x17, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x5f, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, - 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, - 0x6d, 0x61, 0x6c, 0x52, 0x15, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, - 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x62, 0x69, - 0x6e, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x30, 0x18, 0x18, 0x20, 0x01, 0x28, + 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x76, 0x65, 0x72, + 0x61, 0x67, 0x65, 0x12, 0x52, 0x0a, 0x16, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, + 0x6c, 0x52, 0x14, 0x6d, 0x61, 0x78, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x41, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x69, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x10, 0x65, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x6c, + 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x73, 0x5f, 0x74, + 0x6f, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x73, 0x54, 0x6f, 0x4d, 0x61, 0x78, 0x12, 0x43, 0x0a, 0x1e, 0x66, 0x75, 0x6c, 0x6c, + 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x66, + 0x66, 0x69, 0x63, 0x5f, 0x63, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x1b, 0x66, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x43, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x12, 0x52, 0x0a, + 0x16, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x68, 0x61, 0x6c, 0x66, 0x77, 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x6d, 0x69, 0x6e, + 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x6c, 0x66, 0x77, 0x61, + 0x79, 0x12, 0x41, 0x0a, 0x1d, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x63, 0x75, 0x74, 0x6f, + 0x66, 0x66, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1a, 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, + 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x43, 0x75, + 0x74, 0x6f, 0x66, 0x66, 0x12, 0x52, 0x0a, 0x16, 0x6d, 0x69, 0x73, 0x73, 0x5f, 0x70, 0x65, 0x72, + 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, + 0x61, 0x6c, 0x52, 0x14, 0x6d, 0x69, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, + 0x67, 0x65, 0x43, 0x75, 0x74, 0x6f, 0x66, 0x66, 0x12, 0x50, 0x0a, 0x15, 0x6d, 0x69, 0x73, 0x73, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x6e, 0x61, 0x6c, 0x74, + 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, + 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x13, 0x6d, 0x69, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x73, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x12, 0x31, 0x0a, 0x14, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x45, 0x78, 0x70, 0x69, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, + 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x61, 0x64, 0x76, 0x61, 0x6e, + 0x63, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x41, 0x64, 0x76, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x1d, 0x65, 0x73, + 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x5f, 0x70, + 0x65, 0x72, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6b, 0x62, 0x18, 0x0f, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x19, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, + 0x74, 0x73, 0x50, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4b, 0x62, 0x12, 0x5c, 0x0a, 0x1b, + 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x72, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, - 0x0b, 0x62, 0x69, 0x6e, 0x6f, 0x6d, 0x54, 0x65, 0x73, 0x74, 0x50, 0x30, 0x3a, 0x04, 0xe8, 0xa0, - 0x1f, 0x01, 0x22, 0xd8, 0x03, 0x0a, 0x0e, 0x50, 0x6f, 0x43, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x6d, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x03, 0x64, 0x69, 0x6d, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x5f, 0x6c, 0x61, 0x79, - 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6e, 0x4c, 0x61, 0x79, 0x65, - 0x72, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x06, 0x6e, 0x48, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1c, 0x0a, 0x0a, 0x6e, - 0x5f, 0x6b, 0x76, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x08, 0x6e, 0x4b, 0x76, 0x48, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x63, - 0x61, 0x62, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x76, - 0x6f, 0x63, 0x61, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4a, 0x0a, 0x12, 0x66, 0x66, 0x6e, 0x5f, - 0x64, 0x69, 0x6d, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x69, 0x65, 0x72, 0x18, 0x06, + 0x19, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x52, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x12, 0x67, 0x0a, 0x21, 0x62, 0x61, + 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x6e, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, + 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, + 0x6d, 0x61, 0x6c, 0x52, 0x1e, 0x62, 0x61, 0x64, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x61, 0x74, 0x65, 0x12, 0x56, 0x0a, 0x18, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, + 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, + 0x6d, 0x61, 0x6c, 0x52, 0x16, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x56, 0x0a, 0x18, 0x64, + 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x67, 0x6f, 0x6f, 0x64, 0x5f, 0x70, 0x65, 0x72, + 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x16, 0x64, 0x6f, 0x77, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x47, 0x6f, 0x6f, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, + 0x61, 0x67, 0x65, 0x12, 0x54, 0x0a, 0x17, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x62, 0x61, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, - 0x61, 0x6c, 0x52, 0x10, 0x66, 0x66, 0x6e, 0x44, 0x69, 0x6d, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x70, - 0x6c, 0x69, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, - 0x5f, 0x6f, 0x66, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x75, 0x6c, 0x74, 0x69, - 0x70, 0x6c, 0x65, 0x4f, 0x66, 0x12, 0x37, 0x0a, 0x08, 0x6e, 0x6f, 0x72, 0x6d, 0x5f, 0x65, 0x70, - 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, + 0x61, 0x6c, 0x52, 0x15, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x61, 0x64, 0x50, + 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, 0x4e, 0x0a, 0x14, 0x64, 0x6f, 0x77, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, + 0x64, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, - 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x07, 0x6e, 0x6f, 0x72, 0x6d, 0x45, 0x70, 0x73, 0x12, 0x1d, - 0x0a, 0x0a, 0x72, 0x6f, 0x70, 0x65, 0x5f, 0x74, 0x68, 0x65, 0x74, 0x61, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x09, 0x72, 0x6f, 0x70, 0x65, 0x54, 0x68, 0x65, 0x74, 0x61, 0x12, 0x26, 0x0a, - 0x0f, 0x75, 0x73, 0x65, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x6f, 0x70, 0x65, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x75, 0x73, 0x65, 0x53, 0x63, 0x61, 0x6c, 0x65, - 0x64, 0x52, 0x6f, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x65, 0x71, 0x5f, 0x6c, 0x65, 0x6e, - 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x65, 0x71, 0x4c, 0x65, 0x6e, 0x12, 0x37, - 0x0a, 0x08, 0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x07, - 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xe5, 0x01, - 0x0a, 0x11, 0x50, 0x6f, 0x43, 0x53, 0x74, 0x61, 0x74, 0x54, 0x65, 0x73, 0x74, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, - 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x74, 0x54, - 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3b, 0x0a, 0x0a, 0x70, 0x5f, 0x6d, 0x69, - 0x73, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, + 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x12, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x48, + 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x5e, 0x0a, 0x1c, 0x64, 0x6f, 0x77, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x1a, 0x64, + 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x12, 0x54, 0x0a, 0x17, 0x71, 0x75, 0x69, + 0x63, 0x6b, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x15, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x46, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, + 0x40, 0x0a, 0x0d, 0x62, 0x69, 0x6e, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x30, + 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, + 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0b, 0x62, 0x69, 0x6e, 0x6f, 0x6d, 0x54, 0x65, 0x73, 0x74, 0x50, + 0x30, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xd8, 0x03, 0x0a, 0x0e, 0x50, 0x6f, 0x43, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, + 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x64, 0x69, 0x6d, 0x12, 0x19, 0x0a, 0x08, + 0x6e, 0x5f, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, + 0x6e, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x5f, 0x68, 0x65, 0x61, + 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6e, 0x48, 0x65, 0x61, 0x64, 0x73, + 0x12, 0x1c, 0x0a, 0x0a, 0x6e, 0x5f, 0x6b, 0x76, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x6e, 0x4b, 0x76, 0x48, 0x65, 0x61, 0x64, 0x73, 0x12, 0x1d, + 0x0a, 0x0a, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x09, 0x76, 0x6f, 0x63, 0x61, 0x62, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4a, 0x0a, + 0x12, 0x66, 0x66, 0x6e, 0x5f, 0x64, 0x69, 0x6d, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, + 0x69, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x10, 0x66, 0x66, 0x6e, 0x44, 0x69, 0x6d, 0x4d, + 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x69, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x75, 0x6c, + 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x6f, 0x66, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, + 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x4f, 0x66, 0x12, 0x37, 0x0a, 0x08, 0x6e, 0x6f, + 0x72, 0x6d, 0x5f, 0x65, 0x70, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x09, 0x70, 0x4d, 0x69, 0x73, - 0x6d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x48, 0x0a, 0x11, 0x70, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x07, 0x6e, 0x6f, 0x72, 0x6d, + 0x45, 0x70, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x6f, 0x70, 0x65, 0x5f, 0x74, 0x68, 0x65, 0x74, + 0x61, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x72, 0x6f, 0x70, 0x65, 0x54, 0x68, 0x65, + 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0f, 0x75, 0x73, 0x65, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x64, + 0x5f, 0x72, 0x6f, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x75, 0x73, 0x65, + 0x53, 0x63, 0x61, 0x6c, 0x65, 0x64, 0x52, 0x6f, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x65, + 0x71, 0x5f, 0x6c, 0x65, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x65, 0x71, + 0x4c, 0x65, 0x6e, 0x12, 0x37, 0x0a, 0x08, 0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, + 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, + 0x6d, 0x61, 0x6c, 0x52, 0x07, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x3a, 0x04, 0xe8, 0xa0, + 0x1f, 0x01, 0x22, 0xe5, 0x01, 0x0a, 0x11, 0x50, 0x6f, 0x43, 0x53, 0x74, 0x61, 0x74, 0x54, 0x65, + 0x73, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x74, + 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, - 0x70, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x3a, - 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x9d, 0x05, 0x0a, 0x09, 0x50, 0x6f, 0x63, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x64, - 0x69, 0x66, 0x66, 0x69, 0x63, 0x75, 0x6c, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x44, 0x69, 0x66, 0x66, 0x69, 0x63, 0x75, 0x6c, - 0x74, 0x79, 0x12, 0x34, 0x0a, 0x16, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x14, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x61, - 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x46, 0x0a, 0x20, 0x70, 0x6f, 0x63, 0x5f, - 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x70, 0x6f, - 0x63, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x1c, 0x70, 0x6f, 0x63, 0x44, 0x61, 0x74, 0x61, 0x50, 0x72, 0x75, 0x6e, 0x69, - 0x6e, 0x67, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, - 0x12, 0x4c, 0x0a, 0x13, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x11, 0x77, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x4a, - 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x50, 0x6f, 0x43, 0x4d, 0x6f, - 0x64, 0x65, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0b, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x65, 0x71, 0x5f, 0x6c, 0x65, 0x6e, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x65, 0x71, 0x4c, 0x65, 0x6e, 0x12, 0x24, - 0x0a, 0x0e, 0x70, 0x6f, 0x63, 0x5f, 0x76, 0x32, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, 0x6f, 0x63, 0x56, 0x32, 0x45, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x6f, 0x63, 0x5f, 0x76, 0x32, 0x5f, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x63, 0x56, 0x32, 0x45, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x12, 0x43, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x74, 0x65, 0x73, 0x74, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x50, 0x6f, 0x43, - 0x53, 0x74, 0x61, 0x74, 0x54, 0x65, 0x73, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x08, - 0x73, 0x74, 0x61, 0x74, 0x54, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x6c, 0x6f, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6c, - 0x6f, 0x74, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x70, 0x6f, 0x63, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, - 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x70, 0x6f, 0x63, 0x4e, 0x6f, 0x72, 0x6d, 0x61, - 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x3a, - 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x41, 0x0a, 0x07, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x8b, 0x04, 0x0a, 0x10, 0x43, 0x6f, 0x6c, - 0x6c, 0x61, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x52, 0x0a, - 0x16, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x14, 0x73, 0x6c, 0x61, - 0x73, 0x68, 0x46, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x12, 0x54, 0x0a, 0x17, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, - 0x52, 0x15, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x46, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, - 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x6d, 0x0a, 0x24, 0x64, 0x6f, 0x77, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x6d, 0x69, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, - 0x6e, 0x74, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0d, + 0x64, 0x69, 0x73, 0x74, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3b, 0x0a, + 0x0a, 0x70, 0x5f, 0x6d, 0x69, 0x73, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, + 0x09, 0x70, 0x4d, 0x69, 0x73, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x48, 0x0a, 0x11, 0x70, 0x5f, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, - 0x6d, 0x61, 0x6c, 0x52, 0x21, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x4d, 0x69, 0x73, - 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x54, 0x68, 0x72, - 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, - 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x67, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, - 0x69, 0x6f, 0x64, 0x45, 0x6e, 0x64, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x48, 0x0a, 0x11, 0x62, - 0x61, 0x73, 0x65, 0x5f, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, - 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x62, 0x61, 0x73, 0x65, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, - 0x52, 0x61, 0x74, 0x69, 0x6f, 0x12, 0x59, 0x0a, 0x1a, 0x63, 0x6f, 0x6c, 0x6c, 0x61, 0x74, 0x65, - 0x72, 0x61, 0x6c, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x75, - 0x6e, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x70, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x9d, 0x05, 0x0a, 0x09, 0x50, + 0x6f, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x5f, 0x64, 0x69, 0x66, 0x66, 0x69, 0x63, 0x75, 0x6c, 0x74, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x44, 0x69, 0x66, + 0x66, 0x69, 0x63, 0x75, 0x6c, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x16, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x46, 0x0a, + 0x20, 0x70, 0x6f, 0x63, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x69, 0x6e, + 0x67, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x70, 0x6f, 0x63, 0x44, 0x61, 0x74, 0x61, + 0x50, 0x72, 0x75, 0x6e, 0x69, 0x6e, 0x67, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x54, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x4c, 0x0a, 0x13, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, + 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, + 0x52, 0x11, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x65, 0x46, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x12, 0x4a, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x63, 0x6f, 0x6c, 0x6c, 0x61, 0x74, 0x65, - 0x72, 0x61, 0x6c, 0x50, 0x65, 0x72, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x55, 0x6e, 0x69, 0x74, - 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xf3, 0x03, 0x0a, 0x13, 0x42, 0x69, 0x74, 0x63, 0x6f, - 0x69, 0x6e, 0x52, 0x65, 0x77, 0x61, 0x72, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x2e, - 0x0a, 0x13, 0x75, 0x73, 0x65, 0x5f, 0x62, 0x69, 0x74, 0x63, 0x6f, 0x69, 0x6e, 0x5f, 0x72, 0x65, - 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x75, 0x73, 0x65, - 0x42, 0x69, 0x74, 0x63, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x77, 0x61, 0x72, 0x64, 0x73, 0x12, 0x30, - 0x0a, 0x14, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x5f, - 0x72, 0x65, 0x77, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x12, 0x69, 0x6e, - 0x69, 0x74, 0x69, 0x61, 0x6c, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x77, 0x61, 0x72, 0x64, - 0x12, 0x3b, 0x0a, 0x0a, 0x64, 0x65, 0x63, 0x61, 0x79, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, - 0x61, 0x6c, 0x52, 0x09, 0x64, 0x65, 0x63, 0x61, 0x79, 0x52, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, - 0x0d, 0x67, 0x65, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x67, 0x65, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x45, 0x70, 0x6f, - 0x63, 0x68, 0x12, 0x56, 0x0a, 0x18, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x62, 0x6f, 0x6e, 0x75, 0x73, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, - 0x61, 0x6c, 0x52, 0x16, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, - 0x6f, 0x6e, 0x75, 0x73, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x59, 0x0a, 0x1a, 0x66, 0x75, - 0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x6f, 0x6e, 0x75, - 0x73, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x50, 0x6f, 0x43, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x02, + 0x18, 0x01, 0x52, 0x0b, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x65, + 0x71, 0x5f, 0x6c, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x65, 0x71, + 0x4c, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x6f, 0x63, 0x5f, 0x76, 0x32, 0x5f, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, 0x6f, 0x63, + 0x56, 0x32, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x1b, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x6f, 0x63, 0x5f, 0x76, 0x32, + 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x63, 0x56, + 0x32, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x43, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, + 0x5f, 0x74, 0x65, 0x73, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x50, 0x6f, 0x43, 0x53, 0x74, 0x61, 0x74, 0x54, 0x65, 0x73, 0x74, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x54, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, + 0x10, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x6c, 0x6f, 0x74, + 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x6c, 0x6f, 0x74, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x70, 0x6f, 0x63, 0x5f, + 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x70, 0x6f, 0x63, + 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x41, 0x0a, 0x07, 0x44, 0x65, + 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x65, + 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, + 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x8b, 0x04, + 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x61, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x12, 0x52, 0x0a, 0x16, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, + 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, + 0x52, 0x14, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x46, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x54, 0x0a, 0x17, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x5f, + 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, + 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x15, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x46, 0x72, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x6d, 0x0a, 0x24, + 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6d, 0x69, 0x73, 0x73, 0x65, 0x64, 0x5f, + 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x21, 0x64, 0x6f, 0x77, 0x6e, 0x74, 0x69, + 0x6d, 0x65, 0x4d, 0x69, 0x73, 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, + 0x67, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x67, + 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x67, 0x72, 0x61, + 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x45, 0x6e, 0x64, 0x45, 0x70, 0x6f, 0x63, 0x68, + 0x12, 0x48, 0x0a, 0x11, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x62, 0x61, 0x73, 0x65, 0x57, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x52, 0x61, 0x74, 0x69, 0x6f, 0x12, 0x59, 0x0a, 0x1a, 0x63, 0x6f, + 0x6c, 0x6c, 0x61, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x77, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x66, 0x75, - 0x6c, 0x6c, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x42, 0x6f, 0x6e, 0x75, 0x73, 0x46, - 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x5f, 0x0a, 0x1d, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x6f, 0x6e, 0x75, 0x73, 0x5f, - 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x1a, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x42, 0x6f, 0x6e, 0x75, 0x73, - 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xb0, 0x04, 0x0a, - 0x14, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x59, 0x0a, 0x1a, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, - 0x74, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x5f, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x63, 0x6f, + 0x6c, 0x6c, 0x61, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x50, 0x65, 0x72, 0x57, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x55, 0x6e, 0x69, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xf3, 0x03, 0x0a, 0x13, + 0x42, 0x69, 0x74, 0x63, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x77, 0x61, 0x72, 0x64, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x65, 0x5f, 0x62, 0x69, 0x74, 0x63, 0x6f, + 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x11, 0x75, 0x73, 0x65, 0x42, 0x69, 0x74, 0x63, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x77, 0x61, + 0x72, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x65, + 0x70, 0x6f, 0x63, 0x68, 0x5f, 0x72, 0x65, 0x77, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, + 0x65, 0x77, 0x61, 0x72, 0x64, 0x12, 0x3b, 0x0a, 0x0a, 0x64, 0x65, 0x63, 0x61, 0x79, 0x5f, 0x72, + 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, - 0x74, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64, - 0x12, 0x59, 0x0a, 0x1a, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x7a, 0x6f, - 0x6e, 0x65, 0x5f, 0x75, 0x70, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, - 0x61, 0x6c, 0x52, 0x17, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5a, 0x6f, 0x6e, - 0x65, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x47, 0x0a, 0x10, 0x70, - 0x72, 0x69, 0x63, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x69, 0x74, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, - 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, - 0x63, 0x69, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x1b, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x75, 0x74, 0x69, 0x6c, 0x69, - 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, 0x13, 0x6d, 0x69, 0x6e, 0x5f, 0x70, 0x65, 0x72, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x10, 0x6d, 0x69, 0x6e, 0x50, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x72, - 0x69, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x14, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x11, 0x62, 0x61, 0x73, 0x65, 0x50, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x50, - 0x72, 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x16, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, - 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x67, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, - 0x64, 0x45, 0x6e, 0x64, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x67, 0x72, 0x61, - 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x18, 0x67, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x50, 0x65, 0x72, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x72, 0x69, 0x63, 0x65, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, - 0xa7, 0x04, 0x0a, 0x15, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x1d, 0x65, 0x73, 0x74, - 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x5f, 0x70, 0x65, - 0x72, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6b, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x19, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, - 0x73, 0x50, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4b, 0x62, 0x12, 0x49, 0x0a, 0x12, 0x6b, - 0x62, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, - 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x6b, 0x62, 0x50, 0x65, 0x72, 0x49, 0x6e, 0x70, 0x75, - 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x4b, 0x0a, 0x13, 0x6b, 0x62, 0x5f, 0x70, 0x65, 0x72, - 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, + 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x09, 0x64, 0x65, 0x63, 0x61, 0x79, 0x52, 0x61, + 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x67, 0x65, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x5f, 0x65, 0x70, + 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x67, 0x65, 0x6e, 0x65, 0x73, + 0x69, 0x73, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x56, 0x0a, 0x18, 0x75, 0x74, 0x69, 0x6c, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x6f, 0x6e, 0x75, 0x73, 0x5f, 0x66, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x16, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x6f, 0x6e, 0x75, 0x73, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, + 0x59, 0x0a, 0x1a, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x62, 0x6f, 0x6e, 0x75, 0x73, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, - 0x6c, 0x52, 0x10, 0x6b, 0x62, 0x50, 0x65, 0x72, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x2f, 0x0a, 0x13, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x12, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x72, - 0x69, 0x6f, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x69, 0x6e, 0x76, 0x61, 0x6c, - 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x65, - 0x72, 0x69, 0x6f, 0x64, 0x12, 0x3a, 0x0a, 0x19, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x63, 0x75, 0x72, 0x76, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x43, 0x75, 0x72, 0x76, 0x65, - 0x12, 0x48, 0x0a, 0x20, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x1e, 0x6d, 0x69, 0x6e, 0x69, - 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x61, - 0x78, 0x5f, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x5f, 0x70, 0x65, 0x72, - 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x15, 0x6d, 0x61, - 0x78, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x50, 0x65, 0x72, 0x42, 0x6c, - 0x6f, 0x63, 0x6b, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xae, 0x02, 0x0a, 0x15, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x43, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x12, 0x47, 0x0a, 0x20, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x70, 0x65, - 0x72, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1d, 0x65, - 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x50, 0x65, 0x72, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x45, 0x0a, 0x0f, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, - 0x6d, 0x61, 0x6c, 0x52, 0x0e, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x12, 0x43, 0x0a, 0x0e, 0x73, 0x6c, 0x61, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0d, 0x73, 0x6c, 0x61, 0x73, 0x68, - 0x46, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x19, 0x75, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x77, - 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x75, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x69, - 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xc9, 0x01, 0x0a, 0x15, 0x47, - 0x65, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x47, 0x75, 0x61, 0x72, 0x64, 0x69, 0x61, 0x6e, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, - 0x6d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, - 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x4d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, - 0x6c, 0x64, 0x12, 0x3d, 0x0a, 0x1b, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x61, - 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x4d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4d, 0x69, 0x6e, 0x48, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x67, 0x75, 0x61, 0x72, 0x64, 0x69, 0x61, 0x6e, 0x5f, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x69, 0x61, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, - 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x8b, 0x01, 0x0a, 0x15, 0x44, 0x65, 0x76, 0x65, 0x6c, - 0x6f, 0x70, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x12, 0x2c, 0x0a, 0x12, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, - 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x75, 0x6e, - 0x74, 0x69, 0x6c, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x3e, - 0x0a, 0x1b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x64, 0x65, 0x76, 0x65, 0x6c, 0x6f, - 0x70, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x44, 0x65, 0x76, 0x65, - 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x3a, 0x04, - 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xd1, 0x02, 0x0a, 0x17, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, - 0x70, 0x61, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x12, 0x58, 0x0a, 0x29, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x25, 0x6e, 0x65, 0x77, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x42, 0x0a, 0x1d, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, - 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x1b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, - 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x3a, - 0x0a, 0x19, 0x75, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, - 0x74, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x17, 0x75, 0x73, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, - 0x74, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x12, 0x56, 0x0a, 0x28, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, - 0x69, 0x73, 0x74, 0x5f, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, - 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x24, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, - 0x73, 0x74, 0x55, 0x6e, 0x74, 0x69, 0x6c, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x5f, 0x0a, 0x19, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, - 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, 0x61, 0x6c, 0x6c, 0x6f, 0x77, - 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x65, 0x73, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x42, 0xb9, 0x01, 0x0a, 0x17, 0x63, 0x6f, - 0x6d, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x42, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x24, 0x63, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x73, 0x64, 0x6b, 0x2e, - 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x2f, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0xa2, 0x02, 0x03, 0x49, 0x49, 0x58, - 0xaa, 0x02, 0x13, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0xca, 0x02, 0x13, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x5c, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0xe2, 0x02, 0x1f, 0x49, - 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5c, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, - 0x14, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x3a, 0x3a, 0x49, 0x6e, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x52, 0x17, 0x66, 0x75, 0x6c, 0x6c, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x42, + 0x6f, 0x6e, 0x75, 0x73, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x5f, 0x0a, 0x1d, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, + 0x6f, 0x6e, 0x75, 0x73, 0x5f, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, + 0x1a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, + 0x42, 0x6f, 0x6e, 0x75, 0x73, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, + 0x01, 0x22, 0xb0, 0x04, 0x0a, 0x14, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x50, 0x72, 0x69, + 0x63, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x59, 0x0a, 0x1a, 0x73, 0x74, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x6c, 0x6f, 0x77, + 0x65, 0x72, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x73, 0x74, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x4c, 0x6f, 0x77, 0x65, 0x72, + 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x59, 0x0a, 0x1a, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x75, 0x70, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x17, 0x73, 0x74, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64, + 0x12, 0x47, 0x0a, 0x10, 0x70, 0x72, 0x69, 0x63, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, + 0x63, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x45, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x69, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x1b, 0x75, 0x74, 0x69, + 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, + 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, 0x13, 0x6d, 0x69, 0x6e, + 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x6d, 0x69, 0x6e, 0x50, 0x65, 0x72, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x14, 0x62, 0x61, 0x73, 0x65, + 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x62, 0x61, 0x73, 0x65, 0x50, 0x65, 0x72, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x33, 0x0a, 0x16, 0x67, 0x72, 0x61, + 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x65, 0x70, + 0x6f, 0x63, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x67, 0x72, 0x61, 0x63, 0x65, + 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x45, 0x6e, 0x64, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, + 0x0a, 0x1c, 0x67, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, + 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x18, 0x67, 0x72, 0x61, 0x63, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, + 0x64, 0x50, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x72, 0x69, 0x63, 0x65, 0x3a, 0x04, + 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xa7, 0x04, 0x0a, 0x15, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, + 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x40, + 0x0a, 0x1d, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6b, 0x62, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x50, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4b, 0x62, + 0x12, 0x49, 0x0a, 0x12, 0x6b, 0x62, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, + 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0f, 0x6b, 0x62, 0x50, 0x65, + 0x72, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x4b, 0x0a, 0x13, 0x6b, + 0x62, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, + 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x10, 0x6b, 0x62, 0x50, 0x65, 0x72, 0x4f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2f, 0x0a, 0x13, 0x69, 0x6e, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x12, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x69, 0x6e, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x73, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, + 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x3a, 0x0a, 0x19, 0x69, 0x6e, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x5f, 0x63, 0x75, 0x72, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x69, 0x6e, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x43, 0x75, 0x72, 0x76, 0x65, 0x12, 0x48, 0x0a, 0x20, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, + 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x1e, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x74, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x37, 0x0a, 0x18, 0x6d, 0x61, 0x78, 0x5f, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, + 0x50, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xae, + 0x02, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, + 0x6f, 0x43, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x47, 0x0a, 0x20, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x1d, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x50, 0x65, 0x72, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x45, 0x0a, 0x0f, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0e, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x54, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x43, 0x0a, 0x0e, 0x73, 0x6c, 0x61, 0x73, + 0x68, 0x5f, 0x66, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x44, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x6c, 0x52, 0x0d, + 0x73, 0x6c, 0x61, 0x73, 0x68, 0x46, 0x72, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, + 0x19, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x17, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, + 0xc9, 0x01, 0x0a, 0x15, 0x47, 0x65, 0x6e, 0x65, 0x73, 0x69, 0x73, 0x47, 0x75, 0x61, 0x72, 0x64, + 0x69, 0x61, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x6e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x68, + 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x54, 0x68, + 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3d, 0x0a, 0x1b, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x5f, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, + 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x74, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4d, 0x69, 0x6e, + 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x67, 0x75, 0x61, 0x72, 0x64, 0x69, + 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x11, 0x67, 0x75, 0x61, 0x72, 0x64, 0x69, 0x61, 0x6e, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x65, 0x73, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x8b, 0x01, 0x0a, 0x15, + 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x10, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x64, + 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x73, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0xd1, 0x02, 0x0a, 0x17, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x58, 0x0a, 0x29, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x25, 0x6e, 0x65, 0x77, 0x50, 0x61, 0x72, + 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, + 0x42, 0x0a, 0x1d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x1b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x75, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x75, 0x73, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x12, + 0x56, 0x0a, 0x28, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x5f, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x24, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x41, 0x6c, + 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x55, 0x6e, 0x74, 0x69, 0x6c, 0x42, 0x6c, 0x6f, 0x63, + 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x22, 0x5f, 0x0a, + 0x19, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x61, 0x6c, + 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, + 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x3a, 0x04, 0xe8, 0xa0, 0x1f, 0x01, 0x42, 0xb9, + 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x42, 0x0b, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x24, 0x63, 0x6f, 0x73, 0x6d, 0x6f, + 0x73, 0x73, 0x64, 0x6b, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x69, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0xa2, + 0x02, 0x03, 0x49, 0x49, 0x58, 0xaa, 0x02, 0x13, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0xca, 0x02, 0x13, 0x49, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5c, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0xe2, 0x02, 0x1f, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5c, 0x49, 0x6e, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0xea, 0x02, 0x14, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x3a, + 0x3a, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/inference-chain/app/upgrades.go b/inference-chain/app/upgrades.go index 1e0eab059..9258802ac 100644 --- a/inference-chain/app/upgrades.go +++ b/inference-chain/app/upgrades.go @@ -13,6 +13,7 @@ import ( slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" "github.com/productscience/inference/app/upgrades/v0_2_10" + "github.com/productscience/inference/app/upgrades/v0_2_11" v0_2_2 "github.com/productscience/inference/app/upgrades/v0_2_2" v0_2_3 "github.com/productscience/inference/app/upgrades/v0_2_3" "github.com/productscience/inference/app/upgrades/v0_2_4" @@ -61,6 +62,7 @@ func (app *App) setupUpgradeHandlers() { app.UpgradeKeeper.SetUpgradeHandler(v0_2_8.UpgradeName, v0_2_8.CreateUpgradeHandler(app.ModuleManager, app.Configurator(), app.InferenceKeeper, app.BlsKeeper, app.DistrKeeper, app.AuthzKeeper)) app.UpgradeKeeper.SetUpgradeHandler(v0_2_9.UpgradeName, v0_2_9.CreateUpgradeHandler(app.ModuleManager, app.Configurator(), app.InferenceKeeper)) app.UpgradeKeeper.SetUpgradeHandler(v0_2_10.UpgradeName, v0_2_10.CreateUpgradeHandler(app.ModuleManager, app.Configurator(), app.InferenceKeeper, app.DistrKeeper)) + app.UpgradeKeeper.SetUpgradeHandler(v0_2_11.UpgradeName, v0_2_11.CreateUpgradeHandler(app.ModuleManager, app.Configurator(), app.InferenceKeeper)) } func (app *App) registerMigrations() { diff --git a/inference-chain/app/upgrades/v0_2_11/constants.go b/inference-chain/app/upgrades/v0_2_11/constants.go new file mode 100644 index 000000000..46c39a30e --- /dev/null +++ b/inference-chain/app/upgrades/v0_2_11/constants.go @@ -0,0 +1,3 @@ +package v0_2_11 + +const UpgradeName = "v0.2.11" diff --git a/inference-chain/app/upgrades/v0_2_11/upgrades.go b/inference-chain/app/upgrades/v0_2_11/upgrades.go new file mode 100644 index 000000000..58135da2d --- /dev/null +++ b/inference-chain/app/upgrades/v0_2_11/upgrades.go @@ -0,0 +1,58 @@ +package v0_2_11 + +import ( + "context" + "errors" + + upgradetypes "cosmossdk.io/x/upgrade/types" + "github.com/cosmos/cosmos-sdk/types/module" + "github.com/productscience/inference/x/inference/keeper" + "github.com/productscience/inference/x/inference/types" +) + +func CreateUpgradeHandler( + mm *module.Manager, + configurator module.Configurator, + k keeper.Keeper, +) upgradetypes.UpgradeHandler { + return func(ctx context.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) { + k.LogInfo("starting upgrade", types.Upgrades, "version", UpgradeName) + + err := setSafetyWindow(ctx, k) + if err != nil { + return nil, err + } + + toVM, err := mm.RunMigrations(ctx, configurator, fromVM) + if err != nil { + return toVM, err + } + + k.LogInfo("successfully upgraded", types.Upgrades, "version", UpgradeName) + return toVM, nil + } +} + +// setSafetyWindow sets the safety_window parameter to 50. +func setSafetyWindow(ctx context.Context, k keeper.Keeper) error { + params, err := k.GetParams(ctx) + if err != nil { + k.LogError("failed to get params during upgrade", types.Upgrades, "error", err) + return err + } + + if params.EpochParams == nil { + k.LogError("epoch params not initialized", types.Upgrades) + return errors.New("EpochParams are nil") + } + + params.EpochParams.ConfirmationPocSafetyWindow = 50 + + if err := k.SetParams(ctx, params); err != nil { + k.LogError("failed to set params with safety window", types.Upgrades, "error", err) + return err + } + + k.LogInfo("set safety window", types.Upgrades, "safety_window", params.EpochParams.ConfirmationPocSafetyWindow) + return nil +} diff --git a/inference-chain/app/upgrades/v0_2_11/upgrades_test.go b/inference-chain/app/upgrades/v0_2_11/upgrades_test.go new file mode 100644 index 000000000..9596adf6d --- /dev/null +++ b/inference-chain/app/upgrades/v0_2_11/upgrades_test.go @@ -0,0 +1,11 @@ +package v0_2_11 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUpgradeName(t *testing.T) { + require.Equal(t, "v0.2.11", UpgradeName) +} diff --git a/inference-chain/proto/inference/inference/params.proto b/inference-chain/proto/inference/inference/params.proto index 256e143d2..df565c5e9 100644 --- a/inference-chain/proto/inference/inference/params.proto +++ b/inference-chain/proto/inference/inference/params.proto @@ -73,6 +73,7 @@ message EpochParams { int64 inference_pruning_max = 12; int64 poc_pruning_max = 13; Decimal poc_slot_allocation = 14; // Fraction of slots allocated to PoC (0.0 to 1.0, default 0.5) + int64 confirmation_poc_safety_window = 15; } message ValidationParams { diff --git a/inference-chain/test_genesis_overrides.json b/inference-chain/test_genesis_overrides.json index de908c519..dd5e38ace 100644 --- a/inference-chain/test_genesis_overrides.json +++ b/inference-chain/test_genesis_overrides.json @@ -28,7 +28,8 @@ "poc_exchange_duration": "1", "poc_validation_delay": "1", "poc_validation_duration": "4", - "set_new_validators_delay": "1" + "set_new_validators_delay": "1", + "confirmation_poc_safety_window": "0" }, "validation_params": { "expiration_blocks": "10", diff --git a/inference-chain/x/inference/module/confirmation_poc.go b/inference-chain/x/inference/module/confirmation_poc.go index fd1df4593..ff9998c53 100644 --- a/inference-chain/x/inference/module/confirmation_poc.go +++ b/inference-chain/x/inference/module/confirmation_poc.go @@ -16,8 +16,6 @@ import ( "github.com/shopspring/decimal" ) -const safetyWindow = 50 - var pocDeviationCoeff = decimal.New(909, -3) // handleConfirmationPoC manages confirmation PoC trigger decisions and phase transitions @@ -138,7 +136,7 @@ func (am AppModule) checkConfirmationPoCTrigger( epochParams.PocValidationDelay + epochParams.PocValidationDuration + epochParams.SetNewValidatorsDelay + - safetyWindow + epochParams.ConfirmationPocSafetyWindow triggerWindowEnd := nextPoCStart - epochParams.InferenceValidationCutoff - confirmationWindowDuration if blockHeight < setNewValidatorsHeight || blockHeight > triggerWindowEnd { diff --git a/inference-chain/x/inference/types/params.go b/inference-chain/x/inference/types/params.go index 535d2966f..def324382 100644 --- a/inference-chain/x/inference/types/params.go +++ b/inference-chain/x/inference/types/params.go @@ -124,6 +124,7 @@ func DefaultEpochParams() *EpochParams { SetNewValidatorsDelay: 1, InferenceValidationCutoff: 0, InferencePruningEpochThreshold: 2, // Number of epochs after which inferences can be pruned + ConfirmationPocSafetyWindow: 50, PocSlotAllocation: &Decimal{ // Default 0.5 (50%) fraction of nodes allocated to PoC slots Value: 5, Exponent: -1, @@ -343,6 +344,9 @@ func (p *EpochParams) Validate() error { if p.InferencePruningEpochThreshold < 1 { return fmt.Errorf("inference pruning epoch threshold must be at least 1") } + if p.ConfirmationPocSafetyWindow < 0 { + return fmt.Errorf("safety window cannot be negative") + } return nil } diff --git a/inference-chain/x/inference/types/params.pb.go b/inference-chain/x/inference/types/params.pb.go index 96174c044..fc2b2e177 100644 --- a/inference-chain/x/inference/types/params.pb.go +++ b/inference-chain/x/inference/types/params.pb.go @@ -445,6 +445,7 @@ type EpochParams struct { InferencePruningMax int64 `protobuf:"varint,12,opt,name=inference_pruning_max,json=inferencePruningMax,proto3" json:"inference_pruning_max,omitempty"` PocPruningMax int64 `protobuf:"varint,13,opt,name=poc_pruning_max,json=pocPruningMax,proto3" json:"poc_pruning_max,omitempty"` PocSlotAllocation *Decimal `protobuf:"bytes,14,opt,name=poc_slot_allocation,json=pocSlotAllocation,proto3" json:"poc_slot_allocation,omitempty"` + ConfirmationPocSafetyWindow int64 `protobuf:"varint,15,opt,name=confirmation_poc_safety_window,json=confirmationPocSafetyWindow,proto3" json:"confirmation_poc_safety_window,omitempty"` } func (m *EpochParams) Reset() { *m = EpochParams{} } @@ -579,6 +580,13 @@ func (m *EpochParams) GetPocSlotAllocation() *Decimal { return nil } +func (m *EpochParams) GetConfirmationPocSafetyWindow() int64 { + if m != nil { + return m.ConfirmationPocSafetyWindow + } + return 0 +} + type ValidationParams struct { FalsePositiveRate *Decimal `protobuf:"bytes,1,opt,name=false_positive_rate,json=falsePositiveRate,proto3" json:"false_positive_rate,omitempty"` MinRampUpMeasurements int32 `protobuf:"varint,2,opt,name=min_ramp_up_measurements,json=minRampUpMeasurements,proto3" json:"min_ramp_up_measurements,omitempty"` @@ -1942,224 +1950,225 @@ func init() { func init() { proto.RegisterFile("inference/inference/params.proto", fileDescriptor_3cf34332021bbe94) } var fileDescriptor_3cf34332021bbe94 = []byte{ - // 3461 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x5a, 0x5b, 0x73, 0x1c, 0x37, - 0x76, 0x36, 0x45, 0x52, 0x24, 0x0f, 0x45, 0x71, 0x88, 0xe1, 0x90, 0xc3, 0x8b, 0x68, 0xd9, 0x6b, - 0x6f, 0x7c, 0x5b, 0xc9, 0xd1, 0x5e, 0x9c, 0xf2, 0xae, 0x55, 0x4b, 0x91, 0xb2, 0xa4, 0xb5, 0x68, - 0x4d, 0x9a, 0x14, 0x37, 0x76, 0xb9, 0xd2, 0x85, 0xe9, 0xc6, 0x0c, 0x51, 0xea, 0x06, 0x5a, 0x0d, - 0x34, 0x2f, 0x7e, 0xcc, 0x63, 0x52, 0x95, 0xca, 0x1f, 0x48, 0x55, 0xde, 0xf2, 0x96, 0xf2, 0xcf, - 0x48, 0xde, 0x36, 0x6f, 0x9b, 0xa7, 0xa4, 0xec, 0x4a, 0x25, 0xef, 0xf9, 0x03, 0x29, 0x1c, 0xa0, - 0xbb, 0x31, 0xc3, 0x21, 0xb7, 0xfd, 0xc2, 0xe2, 0xe0, 0x9c, 0xef, 0x3b, 0xb8, 0x9c, 0x73, 0x70, - 0x00, 0x34, 0xdc, 0xe5, 0x62, 0xc0, 0x72, 0x26, 0x22, 0x76, 0xbf, 0xfe, 0x2f, 0xa3, 0x39, 0x4d, - 0xd5, 0xbd, 0x2c, 0x97, 0x5a, 0x92, 0x76, 0xd5, 0x7e, 0xaf, 0xfa, 0x6f, 0x73, 0x85, 0xa6, 0x5c, - 0xc8, 0xfb, 0xf8, 0xd7, 0xea, 0x6d, 0xae, 0x0e, 0xe5, 0x50, 0xe2, 0xbf, 0xf7, 0xcd, 0x7f, 0xae, - 0x75, 0x23, 0x92, 0x2a, 0x95, 0x2a, 0xb4, 0x02, 0xfb, 0xc3, 0x8a, 0xde, 0xfe, 0x8f, 0x05, 0xb8, - 0xd9, 0x43, 0x4b, 0x64, 0x0f, 0x6e, 0xb1, 0x4c, 0x46, 0x27, 0xa1, 0xb5, 0xdc, 0x9d, 0xba, 0x3b, - 0xf5, 0xde, 0xe2, 0x83, 0xbb, 0xf7, 0x26, 0x98, 0xbe, 0xf7, 0xd8, 0x28, 0x5a, 0x5c, 0xb0, 0xc8, - 0xea, 0x1f, 0x24, 0x80, 0x95, 0x53, 0x9a, 0xf0, 0x98, 0x6a, 0x2e, 0x45, 0xc9, 0x74, 0x03, 0x99, - 0xde, 0x9d, 0xc8, 0x74, 0x5c, 0x69, 0x3b, 0xba, 0xd6, 0xe9, 0x58, 0x0b, 0xf9, 0x0c, 0x20, 0x93, - 0x51, 0x49, 0x36, 0x8d, 0x64, 0x3b, 0x13, 0xc9, 0x7a, 0x32, 0x72, 0x2c, 0x0b, 0x59, 0xf9, 0xaf, - 0xe9, 0x92, 0x96, 0xaf, 0x98, 0x90, 0x29, 0x8f, 0x54, 0xc9, 0x32, 0x73, 0x4d, 0x97, 0x8e, 0x2a, - 0xed, 0xb2, 0x4b, 0x7a, 0xac, 0xc5, 0x70, 0x46, 0x32, 0x49, 0xa8, 0x66, 0x39, 0x4d, 0x4a, 0xce, - 0xd9, 0x6b, 0x38, 0xf7, 0x2a, 0xed, 0x92, 0x33, 0x1a, 0x6b, 0x21, 0xdf, 0x40, 0xa7, 0xcf, 0x75, + // 3486 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x5a, 0x5f, 0x73, 0x1c, 0x37, + 0x72, 0x37, 0xb5, 0xa4, 0x48, 0x36, 0x45, 0x91, 0xc4, 0x72, 0xc9, 0xe5, 0x1f, 0xd1, 0xb2, 0xcf, + 0xbe, 0xf8, 0xdf, 0x49, 0x8e, 0xee, 0x8f, 0x53, 0xbe, 0xb3, 0xea, 0x28, 0x52, 0x96, 0x74, 0x16, + 0xad, 0xcd, 0x90, 0xe2, 0xc5, 0x2e, 0x57, 0xa6, 0xb0, 0x33, 0xd8, 0x25, 0x4a, 0x33, 0xc0, 0x68, + 0x80, 0xe1, 0x1f, 0x7f, 0x84, 0xa4, 0x2a, 0x95, 0x2f, 0x90, 0xaa, 0xbc, 0xe5, 0x2d, 0xe5, 0x6f, + 0x91, 0xe4, 0xed, 0xf2, 0x76, 0x79, 0x4a, 0xca, 0xae, 0x54, 0xf2, 0x9e, 0x2f, 0x90, 0x42, 0x03, + 0x33, 0x83, 0x5d, 0x2e, 0x79, 0xe3, 0x17, 0x16, 0x17, 0xdd, 0xbf, 0x5f, 0xf7, 0x0c, 0xba, 0x1b, + 0x0d, 0x60, 0xe0, 0x2e, 0x17, 0x03, 0x96, 0x33, 0x11, 0xb1, 0xfb, 0xf5, 0x7f, 0x19, 0xcd, 0x69, + 0xaa, 0xee, 0x65, 0xb9, 0xd4, 0x92, 0xb4, 0xab, 0xf1, 0x7b, 0xd5, 0x7f, 0x9b, 0x2b, 0x34, 0xe5, + 0x42, 0xde, 0xc7, 0xbf, 0x56, 0x6f, 0x73, 0x75, 0x28, 0x87, 0x12, 0xff, 0xbd, 0x6f, 0xfe, 0x73, + 0xa3, 0x1b, 0x91, 0x54, 0xa9, 0x54, 0xa1, 0x15, 0xd8, 0x1f, 0x56, 0xf4, 0xf6, 0x7f, 0xcc, 0xc3, + 0xcd, 0x1e, 0x5a, 0x22, 0x7b, 0x70, 0x8b, 0x65, 0x32, 0x3a, 0x09, 0xad, 0xe5, 0xee, 0xd4, 0xdd, + 0xa9, 0xf7, 0x16, 0x1e, 0xdc, 0xbd, 0x37, 0xc1, 0xf4, 0xbd, 0xc7, 0x46, 0xd1, 0xe2, 0x82, 0x05, + 0x56, 0xff, 0x20, 0x01, 0xac, 0x9c, 0xd2, 0x84, 0xc7, 0x54, 0x73, 0x29, 0x4a, 0xa6, 0x1b, 0xc8, + 0xf4, 0xee, 0x44, 0xa6, 0xe3, 0x4a, 0xdb, 0xd1, 0x2d, 0x9f, 0x8e, 0x8d, 0x90, 0xcf, 0x00, 0x32, + 0x19, 0x95, 0x64, 0x2d, 0x24, 0xdb, 0x99, 0x48, 0xd6, 0x93, 0x91, 0x63, 0x99, 0xcf, 0xca, 0x7f, + 0x8d, 0x4b, 0x5a, 0xbe, 0x62, 0x42, 0xa6, 0x3c, 0x52, 0x25, 0xcb, 0xf4, 0x35, 0x2e, 0x1d, 0x55, + 0xda, 0xa5, 0x4b, 0x7a, 0x6c, 0xc4, 0x70, 0x46, 0x32, 0x49, 0xa8, 0x66, 0x39, 0x4d, 0x4a, 0xce, + 0x99, 0x6b, 0x38, 0xf7, 0x2a, 0xed, 0x92, 0x33, 0x1a, 0x1b, 0x21, 0xdf, 0x40, 0xa7, 0xcf, 0x75, 0x24, 0xb9, 0x08, 0x73, 0x76, 0x46, 0xf3, 0xb8, 0xe4, 0xbd, 0x89, 0xbc, 0xef, 0x4d, 0xe4, 0x7d, - 0x64, 0x11, 0x01, 0x02, 0x1c, 0x75, 0xbb, 0x7f, 0xb9, 0x91, 0x84, 0xb0, 0x16, 0x5f, 0x08, 0x9a, - 0xf2, 0x28, 0xcc, 0x72, 0x1e, 0x71, 0x31, 0x2c, 0xe9, 0xe7, 0x90, 0xfe, 0xfd, 0x89, 0xf4, 0xfb, - 0x16, 0xd2, 0xb3, 0x08, 0xc7, 0xbf, 0x1a, 0x4f, 0x68, 0x25, 0x7d, 0x58, 0xef, 0x53, 0x11, 0x9f, - 0xf1, 0x58, 0x9f, 0x84, 0x09, 0x4f, 0xb9, 0xae, 0x26, 0x7b, 0x1e, 0x2d, 0x7c, 0x30, 0x79, 0x00, - 0x25, 0xe6, 0x39, 0x42, 0x9c, 0x89, 0x4e, 0x7f, 0x52, 0xb3, 0xb1, 0x11, 0x49, 0x31, 0xe0, 0x79, - 0xea, 0xfc, 0xab, 0x76, 0x8b, 0x85, 0x6b, 0x6c, 0xec, 0x79, 0x98, 0x9e, 0xdc, 0x2b, 0x6d, 0x44, - 0x23, 0xcd, 0x51, 0x6d, 0x63, 0xc8, 0x04, 0x53, 0x5c, 0x85, 0xc3, 0x82, 0xe6, 0x31, 0xa7, 0x95, - 0x1f, 0xc3, 0x35, 0x36, 0x9e, 0x58, 0xcc, 0x13, 0x07, 0x29, 0x6d, 0x0c, 0x27, 0x35, 0x1b, 0x1b, - 0x31, 0x3b, 0x65, 0x89, 0xcc, 0x58, 0x1e, 0xd2, 0x28, 0x62, 0xaa, 0x9a, 0xab, 0xc5, 0x6b, 0x6c, - 0xec, 0x97, 0x98, 0x5d, 0x84, 0x94, 0x36, 0xe2, 0x49, 0xcd, 0xe4, 0x04, 0x36, 0x32, 0x9a, 0x6b, - 0x1e, 0xf1, 0x8c, 0x0a, 0x3d, 0x66, 0xe5, 0x16, 0x5a, 0xf9, 0x68, 0x72, 0x10, 0xd5, 0xa8, 0x11, - 0x3b, 0xeb, 0xd9, 0x64, 0x01, 0x91, 0xb0, 0xad, 0x73, 0x2a, 0xd4, 0xc0, 0x0c, 0x66, 0xc8, 0x2e, - 0x19, 0x5b, 0x42, 0x63, 0xf7, 0x26, 0xc7, 0x9a, 0x03, 0xee, 0x1a, 0xdc, 0x88, 0xb9, 0x0d, 0x7d, - 0x95, 0xe8, 0xd3, 0x77, 0xff, 0xf7, 0x9f, 0xde, 0x9c, 0xfa, 0xdb, 0xff, 0xf9, 0xee, 0x83, 0xed, - 0x3a, 0x5d, 0x9e, 0x7b, 0xa9, 0xd3, 0xaa, 0xbd, 0xfd, 0xdd, 0x1c, 0xac, 0xb8, 0x65, 0x79, 0x21, - 0x92, 0x0b, 0xd7, 0xdb, 0xb7, 0xe0, 0x96, 0x96, 0x9a, 0x26, 0xa1, 0x2a, 0xb2, 0x2c, 0xb9, 0xc0, - 0x34, 0x37, 0x1d, 0x2c, 0x62, 0xdb, 0x21, 0x36, 0x91, 0x0f, 0x61, 0x45, 0xe6, 0x7c, 0xc8, 0x05, - 0xd5, 0x32, 0x2f, 0xf5, 0x6e, 0xa0, 0x5e, 0xab, 0x16, 0x38, 0xe5, 0x0f, 0x4c, 0x7a, 0xc9, 0xca, - 0x90, 0xa5, 0xa9, 0x2c, 0x84, 0xc6, 0x24, 0x35, 0x1d, 0x2c, 0x6b, 0x99, 0xd9, 0x20, 0xdc, 0xc5, - 0x66, 0xf2, 0x0b, 0x58, 0x53, 0x9a, 0x8a, 0xd8, 0x68, 0x8e, 0x02, 0x66, 0x10, 0xb0, 0x5a, 0x4a, - 0x47, 0x50, 0xbf, 0x86, 0xcd, 0x2c, 0x67, 0x26, 0x7b, 0x0f, 0x73, 0x9a, 0xa6, 0x2c, 0x0e, 0x15, - 0x4d, 0x58, 0x89, 0x9c, 0x45, 0xe4, 0x7a, 0x96, 0xb3, 0x5e, 0xa5, 0x70, 0x48, 0x13, 0xe6, 0xc0, - 0x6f, 0xc2, 0x62, 0xdd, 0x3d, 0x9b, 0x4b, 0x66, 0x03, 0xa8, 0x3a, 0x86, 0xf3, 0x61, 0x47, 0x18, - 0xc6, 0x26, 0xc7, 0x61, 0x3a, 0x58, 0x08, 0x16, 0x6d, 0xdb, 0xbe, 0x69, 0x1a, 0x1b, 0x62, 0xc6, - 0x72, 0x2e, 0x63, 0x0c, 0x6a, 0x7f, 0x88, 0x3d, 0x6c, 0x26, 0x1f, 0x01, 0xf1, 0x75, 0xe9, 0x85, - 0x2c, 0xb4, 0x8d, 0xce, 0x69, 0x93, 0x47, 0x4b, 0x65, 0xdb, 0x4e, 0x1e, 0xc2, 0xf6, 0x65, 0x6d, - 0x63, 0x21, 0x4c, 0xb9, 0x60, 0x39, 0x46, 0xdc, 0x74, 0xd0, 0x1d, 0xc7, 0xf5, 0x58, 0x7e, 0x60, - 0xe4, 0xe4, 0x97, 0xb0, 0xee, 0xe1, 0x53, 0x7a, 0x1e, 0xc6, 0x45, 0x8e, 0x01, 0x8d, 0x81, 0x34, - 0x1d, 0xac, 0x56, 0xd0, 0x03, 0x7a, 0xbe, 0xef, 0x64, 0x24, 0x82, 0x37, 0x8d, 0x2e, 0x17, 0x31, - 0x3f, 0xe5, 0x71, 0x61, 0x52, 0xb8, 0x3c, 0x63, 0xb9, 0x31, 0x1c, 0x31, 0xa1, 0xe9, 0x90, 0xb9, - 0x08, 0xd9, 0xbe, 0x22, 0x0e, 0x23, 0x9e, 0xd2, 0x24, 0xd8, 0x4e, 0xe9, 0xf9, 0xb3, 0x8a, 0xa3, - 0x67, 0x28, 0x7a, 0x15, 0x03, 0xf9, 0x0b, 0xe8, 0x5e, 0x4a, 0x24, 0x4c, 0xd0, 0x7e, 0xc2, 0x62, - 0x0c, 0x89, 0xf9, 0x60, 0x6d, 0x2c, 0x3b, 0x3c, 0xb6, 0x52, 0xf2, 0x0d, 0x7c, 0x78, 0x09, 0x29, - 0x98, 0x3e, 0x93, 0xf9, 0xab, 0x30, 0xa5, 0xba, 0xc8, 0xb9, 0xbe, 0x08, 0xf5, 0x49, 0xce, 0xd4, - 0x89, 0x4c, 0xe2, 0xee, 0x6d, 0x1c, 0xe9, 0x9f, 0x8d, 0x91, 0x7d, 0x69, 0x01, 0x07, 0x4e, 0xff, - 0xa8, 0x54, 0x27, 0xdf, 0xc0, 0xd6, 0x25, 0xf6, 0xb4, 0x48, 0x34, 0xcf, 0x12, 0xce, 0xf2, 0xee, - 0x72, 0x83, 0x81, 0x6f, 0x8c, 0xd9, 0x3a, 0xa8, 0xe0, 0xe4, 0x37, 0xb0, 0x79, 0x89, 0x9d, 0xc6, - 0x71, 0xce, 0x94, 0x62, 0xaa, 0xdb, 0xba, 0x3b, 0xfd, 0xde, 0x42, 0xd0, 0x1d, 0x83, 0xef, 0x96, - 0xf2, 0xb7, 0xff, 0x73, 0x06, 0x5a, 0xe3, 0xdb, 0x2f, 0xf9, 0x1a, 0x36, 0x55, 0xd1, 0x57, 0x3c, - 0xbe, 0x08, 0x73, 0x16, 0x17, 0x11, 0xa6, 0x7e, 0x2e, 0x34, 0xcb, 0x4f, 0x69, 0xe2, 0xca, 0x94, - 0xeb, 0xfb, 0xdb, 0x75, 0xf8, 0xa0, 0x84, 0x3f, 0x73, 0x68, 0x72, 0x0c, 0xdd, 0xcb, 0xdc, 0x2e, - 0xb2, 0x6e, 0x34, 0x60, 0x5e, 0x1b, 0x67, 0x76, 0x61, 0xf7, 0x35, 0x6c, 0x46, 0x45, 0x9e, 0x9b, - 0x64, 0x58, 0xf2, 0x7b, 0xce, 0x35, 0xdd, 0xa4, 0xcf, 0x0e, 0x7f, 0x68, 0xe1, 0x9e, 0x63, 0x7d, - 0x05, 0x9b, 0x7e, 0xc6, 0x49, 0x12, 0x79, 0xc6, 0xe2, 0x70, 0x40, 0x79, 0x52, 0xe4, 0xcc, 0x55, - 0x36, 0xd7, 0x73, 0xaf, 0xd7, 0x89, 0xc9, 0xa2, 0x3f, 0xb7, 0x60, 0xf2, 0x19, 0x6c, 0x19, 0x6a, - 0x0c, 0x3e, 0xdc, 0x5d, 0x5f, 0x17, 0x34, 0xe1, 0x03, 0x1e, 0xd9, 0x98, 0x9a, 0xad, 0xc2, 0x11, - 0xc3, 0xaf, 0x27, 0xa3, 0xbf, 0xf4, 0xe5, 0xe4, 0x1e, 0xb4, 0xd1, 0x49, 0x4f, 0x99, 0xd2, 0x58, - 0x61, 0xd8, 0x54, 0x61, 0x92, 0xce, 0x4c, 0xb0, 0x62, 0x44, 0xc7, 0x56, 0xe2, 0x92, 0xc5, 0x03, - 0xe8, 0xb8, 0x51, 0x8c, 0x21, 0xe6, 0x10, 0xd1, 0xb6, 0xc2, 0x51, 0xcc, 0x27, 0xd0, 0xad, 0xbb, - 0x38, 0x06, 0x9b, 0x47, 0x58, 0xa7, 0xec, 0xdf, 0x08, 0xf0, 0xd3, 0x19, 0xb3, 0x6b, 0xbc, 0xfd, - 0x37, 0x37, 0x61, 0xd1, 0xab, 0x5e, 0x4d, 0xfa, 0xb3, 0x55, 0x6f, 0xc2, 0xc4, 0x50, 0x9f, 0x94, - 0xdb, 0x01, 0xb6, 0x3d, 0xc7, 0x26, 0xf2, 0x3e, 0xb4, 0xac, 0x8a, 0x17, 0x25, 0x76, 0x37, 0x58, - 0xc6, 0x76, 0xcf, 0xfb, 0xdf, 0x04, 0x8b, 0x0c, 0xd5, 0x09, 0x1f, 0x94, 0xdb, 0x00, 0x60, 0xd3, - 0xa1, 0x69, 0x21, 0xbf, 0x85, 0x3b, 0x31, 0x1b, 0xd0, 0x22, 0xd1, 0x61, 0x21, 0xb8, 0x0e, 0xe5, - 0x20, 0x8c, 0x64, 0x9a, 0x15, 0x9a, 0x61, 0x59, 0xc6, 0xdc, 0x46, 0xb0, 0xe1, 0x94, 0x5e, 0x0a, - 0xae, 0x5f, 0x0c, 0xf6, 0xac, 0x86, 0xa9, 0xb7, 0x98, 0x49, 0xb0, 0x66, 0x61, 0x94, 0x71, 0x85, - 0x3a, 0xdb, 0xd9, 0x95, 0x69, 0x65, 0x32, 0x3a, 0x34, 0x82, 0x2a, 0xd3, 0xfd, 0x0a, 0x3a, 0x46, - 0x9b, 0x9d, 0x47, 0x27, 0x54, 0xf8, 0x00, 0xb3, 0x26, 0xd3, 0x8f, 0x6e, 0x74, 0xa7, 0x82, 0x76, - 0x26, 0xa3, 0xc7, 0x4e, 0x5e, 0xe1, 0x3e, 0x86, 0x55, 0x83, 0xf3, 0x6a, 0xf9, 0x98, 0x25, 0xf4, - 0x02, 0x17, 0x66, 0x3a, 0x30, 0x3d, 0xa8, 0x0b, 0xf7, 0x7d, 0x23, 0x21, 0xbf, 0x82, 0xf5, 0x71, - 0x44, 0x69, 0xcb, 0x6e, 0x15, 0x9d, 0x51, 0x50, 0x69, 0xe9, 0x13, 0xe8, 0x2a, 0xa6, 0x43, 0xc1, - 0xce, 0x4a, 0xac, 0xcc, 0x95, 0xb3, 0x66, 0xb7, 0x8d, 0x8e, 0x62, 0xfa, 0x4b, 0x76, 0x76, 0x5c, - 0x49, 0xad, 0xc1, 0x87, 0xb0, 0x55, 0x79, 0xb6, 0x6f, 0x36, 0x2a, 0xb4, 0x1c, 0x0c, 0xdc, 0xd6, - 0xb1, 0x51, 0xa9, 0xd4, 0xa6, 0xf7, 0x50, 0x81, 0x3c, 0x83, 0xb7, 0x6a, 0x7c, 0x96, 0x17, 0xc2, - 0x38, 0x92, 0x5d, 0xbd, 0x3a, 0xb7, 0x2e, 0xa2, 0x47, 0xed, 0x54, 0x8a, 0x3d, 0xab, 0x87, 0x1e, - 0x54, 0xa7, 0xd4, 0x07, 0xd0, 0xb9, 0x4c, 0x95, 0xd2, 0x73, 0xdc, 0x45, 0xa6, 0x83, 0xf6, 0x38, - 0xfc, 0x80, 0x9e, 0x93, 0x9f, 0xc2, 0x32, 0x96, 0xaf, 0x9e, 0xf6, 0x12, 0x6a, 0x2f, 0x99, 0xa3, - 0x4b, 0xad, 0xf7, 0x1c, 0xda, 0xb8, 0xde, 0x89, 0xd4, 0x18, 0xeb, 0x2e, 0x14, 0x6f, 0x37, 0x08, - 0xf3, 0x15, 0xe3, 0x0e, 0x89, 0xd4, 0xbb, 0x15, 0xcc, 0x05, 0xc1, 0xdf, 0xdf, 0x86, 0xd6, 0xf8, - 0xc1, 0xcb, 0x18, 0x1a, 0xd0, 0x44, 0xb1, 0x30, 0x93, 0x8a, 0x6b, 0x7e, 0xca, 0xc2, 0x9c, 0x6a, - 0xd6, 0x28, 0xbf, 0xae, 0x20, 0xb0, 0xe7, 0x70, 0x01, 0xd5, 0xcc, 0x2c, 0x6b, 0x6a, 0x4e, 0x32, - 0x34, 0xcd, 0xc2, 0x22, 0x0b, 0x53, 0x46, 0x55, 0x91, 0xb3, 0x94, 0x09, 0x6d, 0xcf, 0x83, 0xb3, - 0x41, 0x27, 0xe5, 0x22, 0xa0, 0x69, 0xf6, 0x32, 0x3b, 0xf0, 0x84, 0xe4, 0xd7, 0x00, 0x19, 0x55, - 0xca, 0xac, 0x68, 0xd1, 0x2c, 0x53, 0x2e, 0x18, 0xfd, 0x63, 0xa3, 0x4e, 0x02, 0x58, 0x33, 0x56, - 0x3d, 0x6f, 0xa0, 0xa7, 0x2c, 0x37, 0x29, 0xb7, 0x49, 0x5a, 0x5c, 0x4d, 0xb9, 0xa8, 0xa7, 0x65, - 0xd7, 0x22, 0x91, 0x93, 0x9e, 0x4f, 0xe2, 0x9c, 0x6d, 0xc4, 0x49, 0xcf, 0x2f, 0x73, 0x7e, 0x08, - 0x2b, 0xec, 0x3c, 0xe3, 0x36, 0x04, 0xc2, 0x7e, 0x22, 0xa3, 0x57, 0xb6, 0x36, 0x9b, 0x0e, 0x5a, - 0xb5, 0xe0, 0x11, 0xb6, 0x93, 0xb7, 0x61, 0x09, 0xdd, 0x52, 0x85, 0x5a, 0xa2, 0x9f, 0xcc, 0x79, - 0x39, 0x4a, 0x1d, 0x49, 0xe3, 0x25, 0x7b, 0xb0, 0x33, 0x28, 0x92, 0xc4, 0xef, 0xa5, 0xce, 0xe9, - 0x60, 0xc0, 0xa3, 0x32, 0x1e, 0x6c, 0x10, 0x6e, 0x19, 0xad, 0xba, 0x3f, 0x47, 0x56, 0xc7, 0x45, - 0xc4, 0xe5, 0xd9, 0x3b, 0xa1, 0xc9, 0xe0, 0xcc, 0x05, 0xe2, 0x8f, 0x9b, 0xbd, 0xa7, 0x16, 0x49, - 0x76, 0xe1, 0xce, 0x18, 0xe7, 0x58, 0xbf, 0x6c, 0x9c, 0x6e, 0x8e, 0x80, 0x27, 0x74, 0x4b, 0x29, - 0x6f, 0x03, 0x2d, 0xb1, 0x8b, 0xcd, 0xba, 0xa5, 0x54, 0xbd, 0x7b, 0x3a, 0xce, 0x1e, 0x74, 0x90, - 0x33, 0x67, 0xaf, 0x0b, 0xa6, 0xb0, 0xe6, 0x14, 0x34, 0xd1, 0x17, 0x8d, 0xea, 0xbe, 0xb6, 0x81, - 0x06, 0x0e, 0xd9, 0xb3, 0x40, 0xf2, 0xe7, 0xb0, 0xaa, 0x79, 0xca, 0x94, 0x36, 0x1e, 0x5f, 0xaf, - 0xa1, 0x0b, 0xea, 0x76, 0x25, 0x7b, 0x5c, 0x89, 0x8c, 0x17, 0xd4, 0x10, 0x1a, 0x9f, 0x52, 0x11, - 0x31, 0x57, 0xcd, 0xb5, 0x2a, 0xc1, 0xae, 0x6d, 0x37, 0x3b, 0x87, 0xd9, 0xce, 0x52, 0xaa, 0x59, - 0x5c, 0x9d, 0xaf, 0x59, 0x6e, 0x9d, 0x27, 0x7c, 0xd5, 0xc7, 0xc2, 0x6d, 0x26, 0xd8, 0xa8, 0x94, - 0xdc, 0xc9, 0x99, 0xe5, 0xe8, 0x46, 0x5f, 0xf4, 0x4d, 0xe1, 0xc7, 0x05, 0x2e, 0x44, 0x98, 0xb3, - 0xac, 0xd0, 0xee, 0x0c, 0x9d, 0x33, 0xc5, 0xf2, 0x53, 0xd6, 0x6d, 0x35, 0x29, 0xfc, 0x1c, 0x41, - 0x50, 0xe1, 0x7b, 0x0e, 0x4e, 0x86, 0xf0, 0x56, 0x9f, 0xe2, 0x9d, 0x45, 0x75, 0xe6, 0x74, 0xca, - 0xd6, 0x0e, 0x26, 0x93, 0x95, 0x06, 0x36, 0x76, 0xfa, 0x34, 0xf6, 0xce, 0xa0, 0xcf, 0x3c, 0x12, - 0xcc, 0x2c, 0xc7, 0xd0, 0x1d, 0x21, 0xf6, 0xd3, 0x35, 0x69, 0x52, 0xb2, 0xf9, 0xe8, 0xa7, 0x75, - 0x12, 0x3f, 0x86, 0x6e, 0x2c, 0xcf, 0x84, 0x99, 0xf8, 0x70, 0x28, 0x65, 0xec, 0x17, 0x6c, 0xed, - 0x26, 0xbc, 0x25, 0xfa, 0x89, 0x94, 0xb1, 0x57, 0xae, 0x1d, 0xc1, 0x7a, 0xc5, 0x8b, 0x33, 0x54, - 0xd3, 0xae, 0x36, 0xa0, 0xed, 0x94, 0xe0, 0x47, 0xd4, 0x67, 0xfd, 0x12, 0x56, 0x2b, 0x56, 0x7f, - 0x06, 0x3a, 0x0d, 0x28, 0x49, 0x89, 0xf4, 0x46, 0xff, 0xd7, 0xb0, 0x5d, 0xf1, 0x4d, 0xf2, 0x8e, - 0xb5, 0x06, 0xbc, 0x9b, 0x25, 0xc3, 0x04, 0xf7, 0x38, 0x82, 0xf5, 0xd7, 0x05, 0x8f, 0x5e, 0x95, - 0x75, 0xaa, 0xd7, 0xe5, 0xf5, 0x26, 0xb3, 0x80, 0x60, 0x57, 0xa6, 0xd6, 0xbd, 0xfe, 0x2d, 0x2c, - 0xf5, 0xb9, 0x90, 0x69, 0xa8, 0x99, 0xd2, 0x61, 0xf6, 0x71, 0xb7, 0xdb, 0x80, 0x6b, 0x11, 0x21, - 0x47, 0x4c, 0xe9, 0xde, 0xc7, 0x6e, 0x43, 0xfc, 0xe3, 0x34, 0xdc, 0xee, 0xc9, 0xbd, 0x03, 0x19, - 0xb3, 0xf2, 0x3a, 0xae, 0x05, 0xd3, 0x31, 0x4f, 0x71, 0xfb, 0x9b, 0x0d, 0xcc, 0xbf, 0x64, 0x03, - 0xe6, 0x45, 0x98, 0xd0, 0x0b, 0x96, 0x97, 0x5b, 0xd8, 0x9c, 0x78, 0x8e, 0x3f, 0xc9, 0x3a, 0xcc, - 0x89, 0xf0, 0x84, 0xd1, 0xd8, 0xde, 0x4f, 0xce, 0x06, 0x37, 0xc5, 0x53, 0xf3, 0x8b, 0x6c, 0x03, - 0x88, 0xf0, 0xd5, 0xa9, 0x93, 0xcd, 0xa0, 0x6c, 0x5e, 0x7c, 0x71, 0x6a, 0xa5, 0x77, 0x00, 0x4e, - 0x65, 0x44, 0xfb, 0xa1, 0xe2, 0xdf, 0xda, 0xed, 0x64, 0x36, 0x58, 0xc0, 0x96, 0x43, 0xfe, 0x2d, - 0x23, 0xbf, 0x03, 0x32, 0x18, 0x88, 0x30, 0xe6, 0xa9, 0x5f, 0x7a, 0xde, 0x6c, 0x30, 0xc4, 0xd6, - 0x60, 0x20, 0xf6, 0x79, 0x3a, 0x5a, 0x99, 0x3a, 0x0e, 0x16, 0xca, 0x01, 0x6e, 0x21, 0xb3, 0x01, - 0x94, 0x4d, 0x2f, 0x06, 0xe4, 0x13, 0x98, 0x17, 0x32, 0x4f, 0x43, 0x96, 0x95, 0x17, 0x76, 0xd7, - 0x9b, 0x98, 0x33, 0xda, 0x8f, 0x33, 0x1c, 0x44, 0x2e, 0x33, 0xb3, 0xa0, 0x4c, 0x53, 0xdc, 0x29, - 0x66, 0x83, 0x05, 0xd3, 0x72, 0x64, 0x1a, 0x4c, 0x9d, 0x53, 0x28, 0x16, 0xaa, 0x88, 0x26, 0x2c, - 0x0e, 0x4d, 0x3b, 0xa6, 0xfc, 0xf9, 0x60, 0xa9, 0x50, 0xec, 0x10, 0x5b, 0x03, 0x99, 0x31, 0x33, - 0x85, 0x8a, 0xbd, 0x36, 0x65, 0x38, 0xa6, 0xf5, 0xd9, 0xe0, 0xa6, 0x62, 0xaf, 0x9f, 0x33, 0x53, - 0x20, 0xce, 0xe7, 0xa1, 0xa6, 0xf9, 0x90, 0xe9, 0x46, 0xd9, 0x79, 0x2e, 0x3f, 0x42, 0x65, 0xb7, - 0xb4, 0xff, 0x3d, 0x05, 0x2b, 0x3d, 0xb9, 0x77, 0xa8, 0xa9, 0xc6, 0x25, 0x2f, 0x2f, 0xbb, 0x6f, - 0xc7, 0x5c, 0x69, 0xcf, 0x0b, 0x9b, 0xd4, 0x39, 0x4b, 0x06, 0x53, 0x7b, 0x9f, 0x29, 0x55, 0xc2, - 0x94, 0xab, 0x94, 0xea, 0xe8, 0xa4, 0xd1, 0x71, 0x71, 0x21, 0x3b, 0x70, 0xea, 0xe4, 0x29, 0xac, - 0x64, 0xb6, 0xc8, 0xf1, 0x3a, 0xd1, 0xa4, 0xdc, 0x59, 0xce, 0xb0, 0xd6, 0xa9, 0xba, 0xe1, 0xc6, - 0xf9, 0x8f, 0xb3, 0xb0, 0x50, 0xdf, 0x62, 0xfe, 0x0c, 0x48, 0x79, 0xce, 0x88, 0xb9, 0xd9, 0x4b, - 0x0b, 0xb3, 0xb9, 0x59, 0x67, 0x5e, 0x71, 0x92, 0xfd, 0x4a, 0x40, 0x7e, 0x01, 0x6b, 0x5e, 0x46, - 0x55, 0x34, 0x35, 0x6e, 0x82, 0x4e, 0x69, 0x1d, 0x7d, 0xb5, 0x96, 0x1e, 0xa2, 0x10, 0xfd, 0xf3, - 0x73, 0xb8, 0x6b, 0x4a, 0xd3, 0x98, 0x6a, 0x7a, 0x65, 0x01, 0x3d, 0x8d, 0xbb, 0xd2, 0x76, 0x26, - 0xa3, 0x7d, 0xaa, 0xe9, 0xe4, 0xf2, 0xf9, 0x39, 0xb4, 0xcf, 0x18, 0x1f, 0x9e, 0x68, 0xeb, 0x25, - 0xe1, 0x80, 0x46, 0x5a, 0xe6, 0x8d, 0x4a, 0xb6, 0x15, 0x0b, 0x44, 0x3f, 0xfa, 0x1c, 0x61, 0xe4, - 0x77, 0x70, 0x2b, 0x35, 0x71, 0x3c, 0x7a, 0x2d, 0xff, 0x93, 0x2b, 0x1e, 0x0c, 0xfc, 0x98, 0xc7, - 0xe3, 0xd0, 0x62, 0xea, 0x25, 0x81, 0x0d, 0x98, 0xb7, 0x5c, 0xdc, 0x9e, 0x62, 0x17, 0x82, 0x39, - 0xfc, 0xfd, 0x2c, 0xf6, 0xfd, 0xd5, 0xd6, 0x63, 0xa5, 0xbf, 0xbe, 0x03, 0xb7, 0xf1, 0x20, 0xf4, - 0xa0, 0xba, 0xed, 0x99, 0x47, 0x7f, 0xbf, 0x65, 0xce, 0x3f, 0x0f, 0xca, 0x3b, 0x9e, 0xcf, 0x60, - 0xeb, 0xd2, 0x55, 0xb6, 0x07, 0x59, 0x40, 0x48, 0x77, 0xec, 0x8a, 0xba, 0x86, 0xef, 0xc1, 0x82, - 0xd2, 0x54, 0x63, 0xde, 0x73, 0xf7, 0xd2, 0x3f, 0xbd, 0x6a, 0x84, 0xa3, 0xae, 0x1f, 0xcc, 0x2b, - 0xf7, 0xdb, 0x1c, 0x6c, 0xfd, 0x55, 0x4f, 0xa4, 0xb6, 0xf7, 0xcf, 0x4b, 0xc1, 0xb2, 0xb7, 0xde, - 0xa6, 0x99, 0x7c, 0x0a, 0x1b, 0xa6, 0x87, 0x26, 0xe6, 0x69, 0xc2, 0xbf, 0xb5, 0x88, 0xb2, 0xb3, - 0xb7, 0xb0, 0xb3, 0xe6, 0xf8, 0xf7, 0xa5, 0x2f, 0x77, 0x7d, 0x75, 0xfe, 0xb9, 0x0b, 0x73, 0x6e, - 0xd1, 0xc8, 0x2a, 0xcc, 0xda, 0xea, 0xde, 0x1e, 0xb6, 0xed, 0x0f, 0xb2, 0x09, 0xf3, 0xec, 0x3c, - 0x93, 0x82, 0xb9, 0xab, 0x97, 0xd9, 0xa0, 0xfa, 0xed, 0x28, 0xfe, 0x6e, 0x06, 0x5a, 0xe3, 0x0f, - 0x29, 0xa6, 0x3a, 0x54, 0x09, 0x55, 0x27, 0xe1, 0x20, 0xa7, 0xe5, 0xd5, 0x10, 0xf6, 0xbe, 0x51, - 0x44, 0xaf, 0x22, 0xf6, 0x73, 0x07, 0x75, 0xb5, 0x86, 0xd9, 0xac, 0xc6, 0x38, 0xcb, 0x9d, 0xad, - 0x51, 0x94, 0x77, 0x46, 0x48, 0xf7, 0x1d, 0x94, 0xa4, 0xf0, 0x4e, 0xb5, 0xc5, 0x9a, 0x0a, 0x92, - 0xf9, 0xb5, 0xc0, 0x8f, 0x4c, 0x02, 0x6f, 0x95, 0x4c, 0x07, 0x48, 0x54, 0x17, 0x06, 0x75, 0x54, - 0xfd, 0x1c, 0xd6, 0x86, 0x39, 0x35, 0x07, 0x52, 0xbc, 0xff, 0x08, 0x99, 0x88, 0x6d, 0x74, 0x62, - 0x60, 0xcd, 0x04, 0x6d, 0x94, 0xda, 0xcb, 0x91, 0xc7, 0x22, 0xc6, 0x98, 0x34, 0x59, 0xa9, 0x4f, - 0x15, 0x0b, 0x5d, 0x3c, 0x62, 0xa1, 0xda, 0xe8, 0x9c, 0xb3, 0x6c, 0x60, 0xbf, 0x47, 0x54, 0x60, - 0x40, 0xe4, 0x2b, 0xd8, 0xf4, 0x9f, 0xc8, 0x58, 0x5e, 0x72, 0x16, 0x82, 0xeb, 0x46, 0x9b, 0xd8, - 0xba, 0xf7, 0x44, 0xc6, 0x72, 0xcb, 0xfd, 0x52, 0xf0, 0xd2, 0x1b, 0xfe, 0x6f, 0x1a, 0xda, 0x13, - 0x9e, 0xbf, 0xc8, 0x3d, 0x68, 0x9b, 0x0d, 0x67, 0xf4, 0x2d, 0xcd, 0x3e, 0x67, 0xce, 0x07, 0x2b, - 0x85, 0x62, 0x23, 0x20, 0x45, 0x3e, 0x86, 0x55, 0x2e, 0xb8, 0xe6, 0x34, 0x71, 0xc9, 0xcb, 0x22, - 0x70, 0xa5, 0x67, 0x02, 0xe2, 0x64, 0x38, 0x3d, 0x16, 0x62, 0xf2, 0x7e, 0xcc, 0x22, 0x7a, 0x61, - 0x6b, 0xda, 0x46, 0x47, 0x54, 0xd4, 0xc7, 0xf2, 0xf5, 0x27, 0xb0, 0x54, 0x5e, 0x90, 0xfa, 0xab, - 0x71, 0xcb, 0x35, 0xda, 0x65, 0x38, 0x86, 0x6e, 0xa1, 0x79, 0x15, 0x68, 0x7d, 0x29, 0x0a, 0x55, - 0xa6, 0xc5, 0x26, 0xab, 0xb1, 0xe6, 0xa1, 0x1f, 0x19, 0xb0, 0xcb, 0x8d, 0x5f, 0xc1, 0x26, 0x1e, - 0x13, 0x23, 0x69, 0x0f, 0xa2, 0xa3, 0xcc, 0x8d, 0x16, 0xc5, 0xe0, 0xf7, 0x1c, 0xdc, 0xa7, 0x0e, - 0xe1, 0x0e, 0xd6, 0xfe, 0xf4, 0x2a, 0xf6, 0xb9, 0x26, 0x15, 0xa4, 0xa3, 0x98, 0x60, 0xc0, 0xad, - 0xfa, 0x77, 0x33, 0xb0, 0x3a, 0xe9, 0x55, 0xd2, 0x0c, 0x4d, 0x69, 0xda, 0xe7, 0x09, 0xd7, 0x17, - 0xe1, 0xb7, 0x52, 0xb0, 0x30, 0xc1, 0x3b, 0xfd, 0xbe, 0x2c, 0x44, 0xb3, 0x5c, 0xb0, 0x5e, 0xe1, - 0xbf, 0x96, 0x82, 0x3d, 0x37, 0xe8, 0x47, 0x06, 0x3c, 0x81, 0xba, 0xc8, 0xb2, 0x8a, 0xfa, 0xc6, - 0x8f, 0xa6, 0x7e, 0x69, 0xd0, 0x96, 0xfa, 0x09, 0xb4, 0xf0, 0xde, 0x2f, 0x64, 0x09, 0x55, 0xe6, - 0xc8, 0xa3, 0x2f, 0x1a, 0x16, 0x01, 0x06, 0xf5, 0xb8, 0x02, 0x91, 0x87, 0xb0, 0xe5, 0x7b, 0xcc, - 0x19, 0x17, 0xb1, 0x3c, 0xab, 0xaf, 0xe0, 0xac, 0x93, 0x6d, 0x78, 0x2a, 0xbf, 0x47, 0x8d, 0xea, - 0x1a, 0xee, 0x67, 0xd0, 0x36, 0xe7, 0x74, 0x33, 0x2c, 0x7c, 0xed, 0x76, 0xd7, 0x91, 0xb3, 0x88, - 0x6b, 0xa5, 0x5c, 0xf4, 0x58, 0x8e, 0x37, 0xf3, 0xf6, 0x16, 0xf2, 0x3e, 0xac, 0x62, 0x9e, 0x18, - 0xd7, 0x77, 0x57, 0xbd, 0x46, 0x36, 0x0a, 0xb8, 0x3a, 0x1b, 0xcd, 0x5d, 0x9d, 0x8d, 0x1e, 0xc2, - 0xf6, 0x08, 0x68, 0xdc, 0x9a, 0xbd, 0xef, 0xed, 0x7a, 0xd0, 0x11, 0xa3, 0xce, 0x65, 0xfe, 0x79, - 0x06, 0x3a, 0x13, 0x9f, 0x99, 0xff, 0xf4, 0x99, 0x7a, 0xea, 0x4f, 0x9d, 0xa9, 0x9f, 0x01, 0x79, - 0xd5, 0x47, 0x0c, 0x17, 0x59, 0xa1, 0x6d, 0xef, 0x1a, 0xb9, 0xc4, 0xf2, 0xab, 0x7e, 0x8f, 0xe5, - 0xcf, 0x0c, 0x0a, 0x7b, 0x4c, 0xbe, 0x80, 0xb6, 0xa3, 0x92, 0x85, 0xae, 0xb9, 0x9a, 0x78, 0x43, - 0x0b, 0xb9, 0x5e, 0x20, 0xcc, 0x92, 0xdd, 0x87, 0xb6, 0x7f, 0xcc, 0x55, 0x76, 0x74, 0xce, 0x0d, - 0xc8, 0x88, 0x08, 0xc7, 0x64, 0x6f, 0x53, 0x7d, 0x80, 0x2b, 0x02, 0xdd, 0xcd, 0xba, 0xf5, 0x83, - 0x8d, 0x11, 0x15, 0x5b, 0x09, 0xba, 0x6b, 0xf9, 0x4f, 0x61, 0x63, 0x82, 0xc1, 0x30, 0x2a, 0xcc, - 0xe1, 0xd1, 0x7a, 0xc5, 0xfa, 0x65, 0xb3, 0x7b, 0x46, 0x4c, 0x9e, 0xc2, 0xdd, 0x94, 0x0b, 0x9e, - 0x16, 0x69, 0x18, 0x49, 0x51, 0xbe, 0x9b, 0x8c, 0x68, 0xa3, 0x97, 0x2c, 0x05, 0x3b, 0x4e, 0x6f, - 0xaf, 0x52, 0xf3, 0xaf, 0x07, 0x14, 0xde, 0x3a, 0xe2, 0xc3, 0x9e, 0x9b, 0x21, 0x6f, 0x39, 0xcb, - 0xc7, 0x01, 0x7c, 0xb3, 0x2b, 0xc5, 0xe5, 0x4a, 0x3a, 0x4f, 0xf9, 0x97, 0x1b, 0xd0, 0x99, 0xf8, - 0xb1, 0x00, 0x79, 0x02, 0x77, 0xd9, 0x79, 0xc6, 0x22, 0xe3, 0x28, 0x7e, 0x51, 0x66, 0x0d, 0x58, - 0x47, 0xb6, 0xce, 0x72, 0xa7, 0xd4, 0xf3, 0x89, 0x8c, 0x21, 0xeb, 0xd2, 0x8f, 0x61, 0x99, 0x26, - 0xd9, 0x09, 0xf5, 0xf6, 0xfb, 0x26, 0xde, 0x72, 0x1b, 0x41, 0xf5, 0xe6, 0xbe, 0x07, 0xb7, 0x47, - 0x2b, 0x94, 0x46, 0x7e, 0xb2, 0x34, 0x52, 0x98, 0x98, 0x35, 0x2b, 0xb2, 0x61, 0x4e, 0x63, 0x7c, - 0x5c, 0xd6, 0x2c, 0xf2, 0x52, 0x87, 0x7b, 0x88, 0x58, 0x77, 0x0a, 0xbd, 0x4a, 0x6e, 0xf3, 0x86, - 0x9b, 0xb0, 0x7f, 0x9b, 0x82, 0xce, 0xc4, 0x2f, 0x1f, 0xc8, 0x6f, 0x60, 0xf3, 0x9a, 0x27, 0x4b, - 0x5b, 0xf8, 0x75, 0xc5, 0x55, 0x6f, 0x94, 0x9f, 0xc1, 0xd6, 0x25, 0xb4, 0x49, 0x4f, 0x27, 0x58, - 0x04, 0xb8, 0xd7, 0x97, 0x71, 0xf8, 0x01, 0x17, 0x4f, 0x51, 0x6e, 0x4e, 0x3f, 0x13, 0x1e, 0x1f, - 0xa7, 0xf1, 0xf1, 0x71, 0x65, 0x38, 0xfe, 0xea, 0x58, 0x56, 0x97, 0x53, 0xd0, 0x99, 0xf8, 0x85, - 0x05, 0xf9, 0x08, 0x48, 0x21, 0x34, 0x4f, 0x5c, 0x5e, 0x70, 0x9d, 0xb0, 0x63, 0x68, 0xa1, 0x04, - 0x9d, 0xc8, 0x19, 0x7f, 0x08, 0x5b, 0xe5, 0x9b, 0x9c, 0xf7, 0x91, 0x47, 0xd5, 0x8b, 0x1b, 0xd8, - 0x8b, 0x0d, 0xa7, 0x52, 0x1b, 0x1c, 0xeb, 0xcd, 0xbf, 0xdf, 0x80, 0xf5, 0x2b, 0xbe, 0xc4, 0x20, - 0x7f, 0x05, 0xef, 0x0b, 0x76, 0x36, 0x72, 0xd5, 0x96, 0xb3, 0x21, 0x57, 0xda, 0xdd, 0x27, 0x2b, - 0x4d, 0x73, 0x3d, 0xda, 0xcd, 0x77, 0x05, 0x3b, 0xf3, 0xe8, 0x02, 0x4f, 0xfd, 0xd0, 0x68, 0xbb, - 0xbe, 0x3f, 0x82, 0x3b, 0x38, 0x46, 0x36, 0x7a, 0x91, 0x37, 0xde, 0xfb, 0x2d, 0xa7, 0xe4, 0x77, - 0xb0, 0x54, 0x41, 0xaf, 0x32, 0x3b, 0x83, 0x8f, 0x37, 0x83, 0x4d, 0xb8, 0xb2, 0x2f, 0x62, 0xf3, - 0xc1, 0x7a, 0xa1, 0x98, 0x8f, 0x2d, 0xc5, 0xe4, 0x18, 0xde, 0x9b, 0x88, 0x0b, 0x27, 0xcc, 0xbf, - 0x75, 0xd0, 0x77, 0xb2, 0x09, 0x3c, 0x2f, 0xc7, 0xd6, 0xc4, 0xcd, 0x69, 0x08, 0x1b, 0x57, 0x7e, - 0x6f, 0x62, 0x1c, 0xb6, 0x5c, 0xb6, 0xfa, 0x6b, 0x96, 0x6a, 0xdc, 0x53, 0xf6, 0xe1, 0xda, 0x69, - 0x54, 0x2c, 0xa3, 0x8b, 0xf6, 0xe8, 0xc5, 0xbf, 0x7e, 0xbf, 0x33, 0xf5, 0x87, 0xef, 0x77, 0xa6, - 0xfe, 0xeb, 0xfb, 0x9d, 0xa9, 0x7f, 0xf8, 0x61, 0xe7, 0x8d, 0x3f, 0xfc, 0xb0, 0xf3, 0xc6, 0x1f, - 0x7f, 0xd8, 0x79, 0xe3, 0xeb, 0x5f, 0x0e, 0xb9, 0x3e, 0x29, 0xfa, 0xf7, 0x22, 0x99, 0xde, 0xcf, - 0x72, 0x19, 0x17, 0x91, 0x56, 0x11, 0x1f, 0xfb, 0xe4, 0xcf, 0xff, 0x86, 0x45, 0x5f, 0x64, 0x4c, - 0xf5, 0x6f, 0xe2, 0x57, 0x7a, 0x3f, 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x33, 0x14, 0x26, - 0xd5, 0x22, 0x28, 0x00, 0x00, + 0x64, 0x11, 0x01, 0x02, 0x1c, 0x75, 0xbb, 0x7f, 0x79, 0x90, 0x84, 0xb0, 0x16, 0x5f, 0x08, 0x9a, + 0xf2, 0x28, 0xcc, 0x72, 0x1e, 0x71, 0x31, 0x2c, 0xe9, 0x67, 0x91, 0xfe, 0xfd, 0x89, 0xf4, 0xfb, + 0x16, 0xd2, 0xb3, 0x08, 0xc7, 0xbf, 0x1a, 0x4f, 0x18, 0x25, 0x7d, 0x58, 0xef, 0x53, 0x11, 0x9f, + 0xf1, 0x58, 0x9f, 0x84, 0x09, 0x4f, 0xb9, 0xae, 0x5e, 0xf6, 0x1c, 0x5a, 0xf8, 0x60, 0xf2, 0x03, + 0x94, 0x98, 0xe7, 0x08, 0x71, 0x26, 0x3a, 0xfd, 0x49, 0xc3, 0xc6, 0x46, 0x24, 0xc5, 0x80, 0xe7, + 0xa9, 0x8b, 0xaf, 0x3a, 0x2c, 0xe6, 0xaf, 0xb1, 0xb1, 0xe7, 0x61, 0x7a, 0x72, 0xaf, 0xb4, 0x11, + 0x8d, 0x0c, 0x47, 0xb5, 0x8d, 0x21, 0x13, 0x4c, 0x71, 0x15, 0x0e, 0x0b, 0x9a, 0xc7, 0x9c, 0x56, + 0x71, 0x0c, 0xd7, 0xd8, 0x78, 0x62, 0x31, 0x4f, 0x1c, 0xa4, 0xb4, 0x31, 0x9c, 0x34, 0x6c, 0x6c, + 0xc4, 0xec, 0x94, 0x25, 0x32, 0x63, 0x79, 0x48, 0xa3, 0x88, 0xa9, 0xea, 0x5d, 0x2d, 0x5c, 0x63, + 0x63, 0xbf, 0xc4, 0xec, 0x22, 0xa4, 0xb4, 0x11, 0x4f, 0x1a, 0x26, 0x27, 0xb0, 0x91, 0xd1, 0x5c, + 0xf3, 0x88, 0x67, 0x54, 0xe8, 0x31, 0x2b, 0xb7, 0xd0, 0xca, 0x47, 0x93, 0x93, 0xa8, 0x46, 0x8d, + 0xd8, 0x59, 0xcf, 0x26, 0x0b, 0x88, 0x84, 0x6d, 0x9d, 0x53, 0xa1, 0x06, 0xe6, 0x61, 0x86, 0xec, + 0x92, 0xb1, 0x45, 0x34, 0x76, 0x6f, 0x72, 0xae, 0x39, 0xe0, 0xae, 0xc1, 0x8d, 0x98, 0xdb, 0xd0, + 0x57, 0x89, 0x3e, 0x7d, 0xf7, 0x7f, 0xff, 0xf1, 0xcd, 0xa9, 0xbf, 0xf9, 0x9f, 0xef, 0x3e, 0xd8, + 0xae, 0xcb, 0xe5, 0xb9, 0x57, 0x3a, 0xad, 0xda, 0xdb, 0xdf, 0xcd, 0xc2, 0x8a, 0x9b, 0x96, 0x17, + 0x22, 0xb9, 0x70, 0xde, 0xbe, 0x05, 0xb7, 0xb4, 0xd4, 0x34, 0x09, 0x55, 0x91, 0x65, 0xc9, 0x05, + 0x96, 0xb9, 0x56, 0xb0, 0x80, 0x63, 0x87, 0x38, 0x44, 0x3e, 0x84, 0x15, 0x99, 0xf3, 0x21, 0x17, + 0x54, 0xcb, 0xbc, 0xd4, 0xbb, 0x81, 0x7a, 0xcb, 0xb5, 0xc0, 0x29, 0x7f, 0x60, 0xca, 0x4b, 0x56, + 0xa6, 0x2c, 0x4d, 0x65, 0x21, 0x34, 0x16, 0xa9, 0x56, 0xb0, 0xa4, 0x65, 0x66, 0x93, 0x70, 0x17, + 0x87, 0xc9, 0x2f, 0x60, 0x4d, 0x69, 0x2a, 0x62, 0xa3, 0x39, 0x0a, 0x98, 0x46, 0xc0, 0x6a, 0x29, + 0x1d, 0x41, 0xfd, 0x1a, 0x36, 0xb3, 0x9c, 0x99, 0xea, 0x3d, 0xcc, 0x69, 0x9a, 0xb2, 0x38, 0x54, + 0x34, 0x61, 0x25, 0x72, 0x06, 0x91, 0xeb, 0x59, 0xce, 0x7a, 0x95, 0xc2, 0x21, 0x4d, 0x98, 0x03, + 0xbf, 0x09, 0x0b, 0xb5, 0x7b, 0xb6, 0x96, 0xcc, 0x04, 0x50, 0x39, 0x86, 0xef, 0xc3, 0x3e, 0x61, + 0x18, 0x9b, 0x1a, 0x87, 0xe5, 0x60, 0x3e, 0x58, 0xb0, 0x63, 0xfb, 0x66, 0x68, 0xec, 0x11, 0x33, + 0x96, 0x73, 0x19, 0x63, 0x52, 0xfb, 0x8f, 0xd8, 0xc3, 0x61, 0xf2, 0x11, 0x10, 0x5f, 0x97, 0x5e, + 0xc8, 0x42, 0xdb, 0xec, 0x6c, 0x99, 0x3a, 0x5a, 0x2a, 0xdb, 0x71, 0xf2, 0x10, 0xb6, 0x2f, 0x6b, + 0x1b, 0x0b, 0x61, 0xca, 0x05, 0xcb, 0x31, 0xe3, 0x5a, 0x41, 0x77, 0x1c, 0xd7, 0x63, 0xf9, 0x81, + 0x91, 0x93, 0x5f, 0xc2, 0xba, 0x87, 0x4f, 0xe9, 0x79, 0x18, 0x17, 0x39, 0x26, 0x34, 0x26, 0x52, + 0x2b, 0x58, 0xad, 0xa0, 0x07, 0xf4, 0x7c, 0xdf, 0xc9, 0x48, 0x04, 0x6f, 0x1a, 0x5d, 0x2e, 0x62, + 0x7e, 0xca, 0xe3, 0xc2, 0x94, 0x70, 0x79, 0xc6, 0x72, 0x63, 0x38, 0x62, 0x42, 0xd3, 0x21, 0x73, + 0x19, 0xb2, 0x7d, 0x45, 0x1e, 0x46, 0x3c, 0xa5, 0x49, 0xb0, 0x9d, 0xd2, 0xf3, 0x67, 0x15, 0x47, + 0xcf, 0x50, 0xf4, 0x2a, 0x06, 0xf2, 0x17, 0xd0, 0xbd, 0x54, 0x48, 0x98, 0xa0, 0xfd, 0x84, 0xc5, + 0x98, 0x12, 0x73, 0xc1, 0xda, 0x58, 0x75, 0x78, 0x6c, 0xa5, 0xe4, 0x1b, 0xf8, 0xf0, 0x12, 0x52, + 0x30, 0x7d, 0x26, 0xf3, 0x57, 0x61, 0x4a, 0x75, 0x91, 0x73, 0x7d, 0x11, 0xea, 0x93, 0x9c, 0xa9, + 0x13, 0x99, 0xc4, 0xdd, 0xdb, 0xf8, 0xa4, 0x7f, 0x36, 0x46, 0xf6, 0xa5, 0x05, 0x1c, 0x38, 0xfd, + 0xa3, 0x52, 0x9d, 0x7c, 0x03, 0x5b, 0x97, 0xd8, 0xd3, 0x22, 0xd1, 0x3c, 0x4b, 0x38, 0xcb, 0xbb, + 0x4b, 0x0d, 0x1e, 0x7c, 0x63, 0xcc, 0xd6, 0x41, 0x05, 0x27, 0xbf, 0x81, 0xcd, 0x4b, 0xec, 0x34, + 0x8e, 0x73, 0xa6, 0x14, 0x53, 0xdd, 0xe5, 0xbb, 0xad, 0xf7, 0xe6, 0x83, 0xee, 0x18, 0x7c, 0xb7, + 0x94, 0xbf, 0xfd, 0x9f, 0xd3, 0xb0, 0x3c, 0xbe, 0xfc, 0x92, 0xaf, 0x61, 0x53, 0x15, 0x7d, 0xc5, + 0xe3, 0x8b, 0x30, 0x67, 0x71, 0x11, 0x61, 0xe9, 0xe7, 0x42, 0xb3, 0xfc, 0x94, 0x26, 0xae, 0x4d, + 0xb9, 0xde, 0xdf, 0xae, 0xc3, 0x07, 0x25, 0xfc, 0x99, 0x43, 0x93, 0x63, 0xe8, 0x5e, 0xe6, 0x76, + 0x99, 0x75, 0xa3, 0x01, 0xf3, 0xda, 0x38, 0xb3, 0x4b, 0xbb, 0xaf, 0x61, 0x33, 0x2a, 0xf2, 0xdc, + 0x14, 0xc3, 0x92, 0xdf, 0x0b, 0xae, 0x56, 0x13, 0x9f, 0x1d, 0xfe, 0xd0, 0xc2, 0xbd, 0xc0, 0xfa, + 0x0a, 0x36, 0xfd, 0x8a, 0x93, 0x24, 0xf2, 0x8c, 0xc5, 0xe1, 0x80, 0xf2, 0xa4, 0xc8, 0x99, 0xeb, + 0x6c, 0xae, 0xe7, 0x5e, 0xaf, 0x0b, 0x93, 0x45, 0x7f, 0x6e, 0xc1, 0xe4, 0x33, 0xd8, 0x32, 0xd4, + 0x98, 0x7c, 0xb8, 0xba, 0xbe, 0x2e, 0x68, 0xc2, 0x07, 0x3c, 0xb2, 0x39, 0x35, 0x53, 0xa5, 0x23, + 0xa6, 0x5f, 0x4f, 0x46, 0x7f, 0xe9, 0xcb, 0xc9, 0x3d, 0x68, 0x63, 0x90, 0x9e, 0x32, 0xa5, 0xb1, + 0xc3, 0xb0, 0xa5, 0xc2, 0x14, 0x9d, 0xe9, 0x60, 0xc5, 0x88, 0x8e, 0xad, 0xc4, 0x15, 0x8b, 0x07, + 0xd0, 0x71, 0x4f, 0x31, 0x86, 0x98, 0x45, 0x44, 0xdb, 0x0a, 0x47, 0x31, 0x9f, 0x40, 0xb7, 0x76, + 0x71, 0x0c, 0x36, 0x87, 0xb0, 0x4e, 0xe9, 0xdf, 0x08, 0xf0, 0xd3, 0x69, 0xb3, 0x6a, 0xbc, 0xfd, + 0x2f, 0x37, 0x61, 0xc1, 0xeb, 0x5e, 0x4d, 0xf9, 0xb3, 0x5d, 0x6f, 0xc2, 0xc4, 0x50, 0x9f, 0x94, + 0xcb, 0x01, 0x8e, 0x3d, 0xc7, 0x21, 0xf2, 0x3e, 0x2c, 0x5b, 0x15, 0x2f, 0x4b, 0xec, 0x6a, 0xb0, + 0x84, 0xe3, 0x5e, 0xf4, 0xbf, 0x09, 0x16, 0x19, 0xaa, 0x13, 0x3e, 0x28, 0x97, 0x01, 0xc0, 0xa1, + 0x43, 0x33, 0x42, 0x7e, 0x0b, 0x77, 0x62, 0x36, 0xa0, 0x45, 0xa2, 0xc3, 0x42, 0x70, 0x1d, 0xca, + 0x41, 0x18, 0xc9, 0x34, 0x2b, 0x34, 0xc3, 0xb6, 0x8c, 0xb9, 0x85, 0x60, 0xc3, 0x29, 0xbd, 0x14, + 0x5c, 0xbf, 0x18, 0xec, 0x59, 0x0d, 0xd3, 0x6f, 0x31, 0x53, 0x60, 0xcd, 0xc4, 0x28, 0x13, 0x0a, + 0x75, 0xb5, 0xb3, 0x33, 0xb3, 0x9c, 0xc9, 0xe8, 0xd0, 0x08, 0xaa, 0x4a, 0xf7, 0x2b, 0xe8, 0x18, + 0x6d, 0x76, 0x1e, 0x9d, 0x50, 0xe1, 0x03, 0xcc, 0x9c, 0xb4, 0x1e, 0xdd, 0xe8, 0x4e, 0x05, 0xed, + 0x4c, 0x46, 0x8f, 0x9d, 0xbc, 0xc2, 0x7d, 0x0c, 0xab, 0x06, 0xe7, 0xf5, 0xf2, 0x31, 0x4b, 0xe8, + 0x05, 0x4e, 0x4c, 0x2b, 0x30, 0x1e, 0xd4, 0x8d, 0xfb, 0xbe, 0x91, 0x90, 0x5f, 0xc1, 0xfa, 0x38, + 0xa2, 0xb4, 0x65, 0x97, 0x8a, 0xce, 0x28, 0xa8, 0xb4, 0xf4, 0x09, 0x74, 0x15, 0xd3, 0xa1, 0x60, + 0x67, 0x25, 0x56, 0xe6, 0xca, 0x59, 0xb3, 0xcb, 0x46, 0x47, 0x31, 0xfd, 0x25, 0x3b, 0x3b, 0xae, + 0xa4, 0xd6, 0xe0, 0x43, 0xd8, 0xaa, 0x22, 0xdb, 0x37, 0x1b, 0x15, 0x5a, 0x0e, 0x06, 0x6e, 0xe9, + 0xd8, 0xa8, 0x54, 0x6a, 0xd3, 0x7b, 0xa8, 0x40, 0x9e, 0xc1, 0x5b, 0x35, 0x3e, 0xcb, 0x0b, 0x61, + 0x02, 0xc9, 0xce, 0x5e, 0x5d, 0x5b, 0x17, 0x30, 0xa2, 0x76, 0x2a, 0xc5, 0x9e, 0xd5, 0xc3, 0x08, + 0xaa, 0x4b, 0xea, 0x03, 0xe8, 0x5c, 0xa6, 0x4a, 0xe9, 0x39, 0xae, 0x22, 0xad, 0xa0, 0x3d, 0x0e, + 0x3f, 0xa0, 0xe7, 0xe4, 0xa7, 0xb0, 0x84, 0xed, 0xab, 0xa7, 0xbd, 0x88, 0xda, 0x8b, 0x66, 0xeb, + 0x52, 0xeb, 0x3d, 0x87, 0x36, 0xce, 0x77, 0x22, 0x35, 0xe6, 0xba, 0x4b, 0xc5, 0xdb, 0x0d, 0xd2, + 0x7c, 0xc5, 0x84, 0x43, 0x22, 0xf5, 0x6e, 0x05, 0x23, 0x7b, 0xb0, 0x73, 0xa9, 0x83, 0x56, 0x74, + 0xc0, 0xf4, 0x45, 0x78, 0xc6, 0x45, 0x2c, 0xcf, 0xb0, 0xfe, 0xb7, 0x82, 0xad, 0xb1, 0xe6, 0xf8, + 0x10, 0x75, 0x7e, 0x8f, 0x2a, 0x2e, 0x93, 0xfe, 0xee, 0x36, 0x2c, 0x8f, 0xef, 0xde, 0x8c, 0xb7, + 0x03, 0x9a, 0x28, 0x16, 0x66, 0x52, 0x71, 0xcd, 0x4f, 0x59, 0x98, 0x53, 0xcd, 0x1a, 0x15, 0xe9, + 0x15, 0x04, 0xf6, 0x1c, 0x2e, 0xa0, 0x9a, 0x99, 0xd8, 0x48, 0xcd, 0x76, 0x88, 0xa6, 0x59, 0x58, + 0x64, 0x61, 0xca, 0xa8, 0x2a, 0x72, 0x96, 0x32, 0xa1, 0xed, 0xa6, 0x72, 0x26, 0xe8, 0xa4, 0x5c, + 0x04, 0x34, 0xcd, 0x5e, 0x66, 0x07, 0x9e, 0x90, 0xfc, 0x1a, 0x20, 0xa3, 0x4a, 0x99, 0xb0, 0x28, + 0x9a, 0x95, 0xdb, 0x79, 0xa3, 0x7f, 0x6c, 0xd4, 0x49, 0x00, 0x6b, 0xc6, 0xaa, 0x17, 0x52, 0xf4, + 0x94, 0xe5, 0xa6, 0x6e, 0x37, 0xa9, 0xad, 0xab, 0x29, 0x17, 0xf5, 0x6b, 0xd9, 0xb5, 0x48, 0xe4, + 0xa4, 0xe7, 0x93, 0x38, 0x67, 0x1a, 0x71, 0xd2, 0xf3, 0xcb, 0x9c, 0x1f, 0xc2, 0x0a, 0x3b, 0xcf, + 0xb8, 0xcd, 0xa3, 0xb0, 0x9f, 0xc8, 0xe8, 0x95, 0x6d, 0xf0, 0x5a, 0xc1, 0x72, 0x2d, 0x78, 0x84, + 0xe3, 0xe4, 0x6d, 0x58, 0xc4, 0xd8, 0x56, 0xa1, 0x96, 0x18, 0x6c, 0xb3, 0x5e, 0xa1, 0x53, 0x47, + 0xd2, 0x84, 0xda, 0x1e, 0xec, 0x0c, 0x8a, 0x24, 0xf1, 0xbd, 0xd4, 0x39, 0x1d, 0x0c, 0x78, 0x54, + 0x26, 0x95, 0xcd, 0xe4, 0x2d, 0xa3, 0x55, 0xfb, 0x73, 0x64, 0x75, 0x5c, 0x5a, 0x5d, 0x7e, 0x7b, + 0x27, 0x34, 0x19, 0x9c, 0xb9, 0x6c, 0xfe, 0x71, 0x6f, 0xef, 0xa9, 0x45, 0x92, 0x5d, 0xb8, 0x33, + 0xc6, 0x39, 0xe6, 0x97, 0x4d, 0xf6, 0xcd, 0x11, 0xf0, 0x04, 0xb7, 0x94, 0xf2, 0x56, 0xe1, 0x12, + 0xbb, 0xd0, 0xcc, 0x2d, 0xa5, 0xea, 0x25, 0xd8, 0x71, 0xf6, 0xa0, 0x83, 0x9c, 0x39, 0x7b, 0x5d, + 0x30, 0x85, 0x8d, 0xab, 0xa0, 0x89, 0xbe, 0x68, 0xd4, 0x3c, 0xb6, 0x0d, 0x34, 0x70, 0xc8, 0x9e, + 0x05, 0x92, 0x3f, 0x87, 0x55, 0xcd, 0x53, 0xa6, 0xb4, 0x89, 0xf8, 0x7a, 0x0e, 0x5d, 0x65, 0x68, + 0x57, 0xb2, 0xc7, 0x95, 0xc8, 0x44, 0x41, 0x0d, 0xa1, 0xf1, 0x29, 0x15, 0x11, 0x73, 0x2d, 0xe1, + 0x72, 0x25, 0xd8, 0xb5, 0xe3, 0x66, 0xf9, 0x31, 0x6b, 0x62, 0x4a, 0x35, 0x8b, 0xab, 0x4d, 0x3a, + 0xcb, 0x6d, 0xf0, 0x84, 0xaf, 0xfa, 0x98, 0xfd, 0xd3, 0xc1, 0x46, 0xa5, 0xe4, 0xb6, 0xdf, 0x2c, + 0xc7, 0x30, 0xfa, 0xa2, 0x6f, 0xba, 0x47, 0x2e, 0x70, 0x22, 0xc2, 0x9c, 0x65, 0x85, 0x76, 0x65, + 0x24, 0x67, 0x8a, 0xe5, 0xa7, 0xac, 0xbb, 0xdc, 0xa4, 0x7b, 0x74, 0x04, 0x41, 0x85, 0xef, 0x39, + 0x38, 0x19, 0xc2, 0x5b, 0x7d, 0x8a, 0x07, 0x1f, 0xd5, 0xc6, 0xd5, 0x29, 0x5b, 0x3b, 0x58, 0x4c, + 0x56, 0x1a, 0xd8, 0xd8, 0xe9, 0xd3, 0xd8, 0xdb, 0xc8, 0x3e, 0xf3, 0x48, 0xb0, 0xb2, 0x1c, 0x43, + 0x77, 0x84, 0xd8, 0xaf, 0xf9, 0xa4, 0x49, 0xdf, 0xe7, 0xa3, 0x9f, 0xd6, 0x2b, 0xc1, 0x31, 0x74, + 0x63, 0x79, 0x26, 0xcc, 0x8b, 0x0f, 0x87, 0x52, 0xc6, 0x7e, 0xd7, 0xd7, 0x6e, 0xc2, 0x5b, 0xa2, + 0x9f, 0x48, 0x19, 0x7b, 0x3d, 0xdf, 0x11, 0xac, 0x57, 0xbc, 0xf8, 0x86, 0x6a, 0xda, 0xd5, 0x06, + 0xb4, 0x9d, 0x12, 0xfc, 0x88, 0xfa, 0xac, 0x5f, 0xc2, 0x6a, 0xc5, 0xea, 0xbf, 0x81, 0x4e, 0x03, + 0x4a, 0x52, 0x22, 0xbd, 0xa7, 0xff, 0x6b, 0xd8, 0xae, 0xf8, 0x26, 0x45, 0xc7, 0x5a, 0x03, 0xde, + 0xcd, 0x92, 0x61, 0x42, 0x78, 0x1c, 0xc1, 0xfa, 0xeb, 0x82, 0x47, 0xaf, 0xca, 0x66, 0xd7, 0x73, + 0x79, 0xbd, 0xc9, 0x5b, 0x40, 0xb0, 0xeb, 0x75, 0x6b, 0xaf, 0x7f, 0x0b, 0x8b, 0x7d, 0x2e, 0x64, + 0x1a, 0x6a, 0xa6, 0x74, 0x98, 0x7d, 0xdc, 0xed, 0x36, 0xe0, 0x5a, 0x40, 0xc8, 0x11, 0x53, 0xba, + 0xf7, 0xb1, 0x5b, 0x10, 0xff, 0xd8, 0x82, 0xdb, 0x3d, 0xb9, 0x77, 0x20, 0x63, 0x56, 0x9e, 0xe9, + 0x2d, 0x43, 0x2b, 0xe6, 0x29, 0x2e, 0x7f, 0x33, 0x81, 0xf9, 0x97, 0x6c, 0xc0, 0x9c, 0x08, 0x13, + 0x7a, 0xc1, 0xf2, 0x72, 0x09, 0x9b, 0x15, 0xcf, 0xf1, 0x27, 0x59, 0x87, 0x59, 0x11, 0x9e, 0x30, + 0x1a, 0xdb, 0x43, 0xce, 0x99, 0xe0, 0xa6, 0x78, 0x6a, 0x7e, 0x91, 0x6d, 0x00, 0x11, 0xbe, 0x3a, + 0x75, 0xb2, 0x69, 0x94, 0xcd, 0x89, 0x2f, 0x4e, 0xad, 0xf4, 0x0e, 0xc0, 0xa9, 0x8c, 0x68, 0x3f, + 0x54, 0xfc, 0x5b, 0xbb, 0x9c, 0xcc, 0x04, 0xf3, 0x38, 0x72, 0xc8, 0xbf, 0x65, 0xe4, 0x77, 0x40, + 0x06, 0x03, 0x11, 0xc6, 0x3c, 0xf5, 0xfb, 0xd7, 0x9b, 0x0d, 0x1e, 0x71, 0x79, 0x30, 0x10, 0xfb, + 0x3c, 0x1d, 0x6d, 0x6f, 0x1d, 0x07, 0x0b, 0xe5, 0x00, 0x97, 0x90, 0x99, 0x00, 0xca, 0xa1, 0x17, + 0x03, 0xf2, 0x09, 0xcc, 0x09, 0x99, 0xa7, 0x21, 0xcb, 0xca, 0x53, 0xbf, 0xeb, 0x4d, 0xcc, 0x1a, + 0xed, 0xc7, 0x19, 0x3e, 0x44, 0x2e, 0x33, 0x33, 0xa1, 0x4c, 0x53, 0x5c, 0x29, 0x66, 0x82, 0x79, + 0x33, 0x72, 0x64, 0x06, 0x4c, 0xb3, 0x54, 0x28, 0x16, 0xaa, 0x88, 0x26, 0x2c, 0x0e, 0xcd, 0x38, + 0x96, 0xfc, 0xb9, 0x60, 0xb1, 0x50, 0xec, 0x10, 0x47, 0x03, 0x99, 0x31, 0xf3, 0x0a, 0x15, 0x7b, + 0x6d, 0x7a, 0x79, 0x2c, 0xeb, 0x33, 0xc1, 0x4d, 0xc5, 0x5e, 0x3f, 0x67, 0xa6, 0xcb, 0x9c, 0xcb, + 0x43, 0x4d, 0xf3, 0x21, 0xd3, 0x8d, 0xaa, 0xf3, 0x6c, 0x7e, 0x84, 0xca, 0x6e, 0x6a, 0xff, 0x7b, + 0x0a, 0x56, 0x7a, 0x72, 0xef, 0x50, 0x53, 0x8d, 0x53, 0x5e, 0x9e, 0x98, 0xdf, 0x8e, 0xb9, 0xd2, + 0x5e, 0x14, 0x36, 0xe9, 0x73, 0x16, 0x0d, 0xa6, 0x8e, 0x3e, 0xd3, 0xaa, 0x84, 0x29, 0x57, 0x29, + 0xd5, 0xd1, 0x49, 0xa3, 0x3d, 0xe7, 0x7c, 0x76, 0xe0, 0xd4, 0xc9, 0x53, 0x58, 0xc9, 0x6c, 0x93, + 0xe3, 0x39, 0xd1, 0xa4, 0xdd, 0x59, 0xca, 0xb0, 0xd7, 0xa9, 0xdc, 0x70, 0xcf, 0xf9, 0x0f, 0x33, + 0x30, 0x5f, 0x1f, 0x85, 0xfe, 0x0c, 0x48, 0xb9, 0x59, 0x89, 0xb9, 0x59, 0x4b, 0x0b, 0xb3, 0xb8, + 0xd9, 0x60, 0x5e, 0x71, 0x92, 0xfd, 0x4a, 0x40, 0x7e, 0x01, 0x6b, 0x5e, 0x45, 0x55, 0x34, 0x35, + 0x61, 0x82, 0x41, 0x69, 0x03, 0x7d, 0xb5, 0x96, 0x1e, 0xa2, 0x10, 0xe3, 0xf3, 0x73, 0xb8, 0x6b, + 0x9a, 0xd0, 0x98, 0x6a, 0x7a, 0x65, 0x17, 0xde, 0xc2, 0x55, 0x69, 0x3b, 0x93, 0xd1, 0x3e, 0xd5, + 0x74, 0x72, 0x0f, 0xfe, 0x1c, 0xda, 0x67, 0x8c, 0x0f, 0x4f, 0xb4, 0x8d, 0x92, 0x70, 0x40, 0x23, + 0x2d, 0xf3, 0x46, 0x2d, 0xdb, 0x8a, 0x05, 0x62, 0x1c, 0x7d, 0x8e, 0x30, 0xf2, 0x3b, 0xb8, 0x95, + 0x9a, 0x3c, 0x1e, 0x3d, 0xdb, 0xff, 0xc9, 0x15, 0xb7, 0x0e, 0x7e, 0xce, 0xe3, 0x9e, 0x6a, 0x21, + 0xf5, 0x8a, 0xc0, 0x06, 0xcc, 0x59, 0x2e, 0x6e, 0xb7, 0xc2, 0xf3, 0xc1, 0x2c, 0xfe, 0x7e, 0x16, + 0xfb, 0xf1, 0x6a, 0xfb, 0xb1, 0x32, 0x5e, 0xdf, 0x81, 0xdb, 0xb8, 0x9b, 0x7a, 0x50, 0x1d, 0x19, + 0xcd, 0x61, 0xbc, 0xdf, 0x32, 0x9b, 0xa8, 0x07, 0xe5, 0x41, 0xd1, 0x67, 0xb0, 0x75, 0xa9, 0x9b, + 0xf7, 0x20, 0xf3, 0x08, 0xe9, 0x8e, 0xb5, 0xf2, 0x35, 0x7c, 0x0f, 0xe6, 0x95, 0xa6, 0x1a, 0xeb, + 0x9e, 0x3b, 0xdc, 0xfe, 0xe9, 0x55, 0x4f, 0x38, 0x1a, 0xfa, 0xc1, 0x9c, 0x72, 0xbf, 0xcd, 0xee, + 0xd8, 0x9f, 0xf5, 0x44, 0x6a, 0x7b, 0x88, 0xbd, 0x18, 0x2c, 0x79, 0xf3, 0x6d, 0x86, 0xc9, 0xa7, + 0xb0, 0x61, 0x3c, 0x34, 0x39, 0x4f, 0x13, 0xfe, 0xad, 0x45, 0x94, 0xce, 0xde, 0x42, 0x67, 0xcd, + 0x1e, 0xf2, 0x4b, 0x5f, 0xee, 0x7c, 0x75, 0xf1, 0xb9, 0x0b, 0xb3, 0x6e, 0xd2, 0xc8, 0x2a, 0xcc, + 0xd8, 0xee, 0xde, 0xee, 0xd8, 0xed, 0x0f, 0xb2, 0x09, 0x73, 0xec, 0x3c, 0x93, 0x82, 0xb9, 0xf3, + 0x9b, 0x99, 0xa0, 0xfa, 0xed, 0x28, 0xfe, 0x76, 0x1a, 0x96, 0xc7, 0x6f, 0x63, 0x4c, 0x77, 0xa8, + 0x12, 0xaa, 0x4e, 0xc2, 0x41, 0x4e, 0xcb, 0xf3, 0x25, 0xf4, 0xbe, 0x51, 0x46, 0xaf, 0x22, 0xf6, + 0x73, 0x07, 0x75, 0xbd, 0x86, 0x59, 0xac, 0xc6, 0x38, 0xcb, 0x95, 0xad, 0x51, 0x96, 0x77, 0x46, + 0x48, 0xf7, 0x1d, 0x94, 0xa4, 0xf0, 0x4e, 0xb5, 0xc4, 0x9a, 0x0e, 0x92, 0xf9, 0xbd, 0xc0, 0x8f, + 0x2c, 0x02, 0x6f, 0x95, 0x4c, 0x07, 0x48, 0x54, 0x37, 0x06, 0x75, 0x56, 0xfd, 0x1c, 0xd6, 0x86, + 0x39, 0x35, 0xbb, 0x5a, 0x3c, 0x44, 0x09, 0x99, 0x88, 0x6d, 0x76, 0x62, 0x62, 0x4d, 0x07, 0x6d, + 0x94, 0xda, 0x13, 0x96, 0xc7, 0x22, 0xc6, 0x9c, 0x34, 0x55, 0xa9, 0x4f, 0x15, 0x0b, 0x5d, 0x3e, + 0x62, 0xa3, 0xda, 0x68, 0x9f, 0xb3, 0x64, 0x60, 0xbf, 0x47, 0x54, 0x60, 0x40, 0xe4, 0x2b, 0xd8, + 0xf4, 0xef, 0xd9, 0x58, 0x5e, 0x72, 0x16, 0x82, 0xeb, 0x46, 0x8b, 0xd8, 0xba, 0x77, 0xcf, 0xc6, + 0x72, 0xcb, 0xfd, 0x52, 0xf0, 0x32, 0x1a, 0xfe, 0xaf, 0x05, 0xed, 0x09, 0x77, 0x68, 0xe4, 0x1e, + 0xb4, 0xcd, 0x82, 0x33, 0x7a, 0x21, 0x67, 0xef, 0x44, 0xe7, 0x82, 0x95, 0x42, 0xb1, 0x11, 0x90, + 0x22, 0x1f, 0xc3, 0x2a, 0x17, 0x5c, 0x73, 0x9a, 0xb8, 0xe2, 0x65, 0x11, 0x38, 0xd3, 0xd3, 0x01, + 0x71, 0x32, 0x7c, 0x3d, 0x16, 0x62, 0xea, 0x7e, 0xcc, 0x22, 0x7a, 0x61, 0x7b, 0xda, 0x46, 0x5b, + 0x54, 0xd4, 0xc7, 0xf6, 0xf5, 0x27, 0xb0, 0x58, 0x9e, 0xb2, 0xfa, 0xb3, 0x71, 0xcb, 0x0d, 0xda, + 0x69, 0x38, 0x86, 0x6e, 0xa1, 0x79, 0x95, 0x68, 0x7d, 0x29, 0x0a, 0x55, 0x96, 0xc5, 0x26, 0xb3, + 0xb1, 0xe6, 0xa1, 0x1f, 0x19, 0xb0, 0xab, 0x8d, 0x5f, 0xc1, 0x26, 0x6e, 0x13, 0x23, 0x69, 0x37, + 0xa2, 0xa3, 0xcc, 0x8d, 0x26, 0xc5, 0xe0, 0xf7, 0x1c, 0xdc, 0xa7, 0x0e, 0xe1, 0x0e, 0xf6, 0xfe, + 0xf4, 0x2a, 0xf6, 0xd9, 0x26, 0x1d, 0xa4, 0xa3, 0x98, 0x60, 0xc0, 0xcd, 0xfa, 0x77, 0xd3, 0xb0, + 0x3a, 0xe9, 0x6a, 0xd3, 0x3c, 0x9a, 0xd2, 0xb4, 0xcf, 0x13, 0xae, 0x2f, 0xc2, 0x6f, 0xa5, 0x60, + 0x61, 0x82, 0x17, 0x03, 0x7d, 0x59, 0x88, 0x66, 0xb5, 0x60, 0xbd, 0xc2, 0x7f, 0x2d, 0x05, 0x7b, + 0x6e, 0xd0, 0x8f, 0x0c, 0x78, 0x02, 0x75, 0x91, 0x65, 0x15, 0xf5, 0x8d, 0x1f, 0x4d, 0xfd, 0xd2, + 0xa0, 0x2d, 0xf5, 0x13, 0x58, 0xc6, 0xc3, 0xc3, 0x90, 0x25, 0x54, 0x99, 0x2d, 0x8f, 0xbe, 0x68, + 0xd8, 0x04, 0x18, 0xd4, 0xe3, 0x0a, 0x44, 0x1e, 0xc2, 0x96, 0x1f, 0x31, 0xf6, 0x44, 0xa8, 0x3e, + 0xc7, 0xb3, 0x41, 0xb6, 0xe1, 0xa9, 0xd8, 0x03, 0xa1, 0xea, 0x2c, 0xef, 0x67, 0xd0, 0x36, 0xfb, + 0x74, 0xf3, 0x58, 0x78, 0x65, 0xee, 0xce, 0x34, 0x67, 0x10, 0xb7, 0x9c, 0x72, 0xd1, 0x63, 0x39, + 0x1e, 0xef, 0xdb, 0xa3, 0xcc, 0xfb, 0xb0, 0x8a, 0x75, 0x62, 0x5c, 0xdf, 0x9d, 0x17, 0x1b, 0xd9, + 0x28, 0xe0, 0xea, 0x6a, 0x34, 0x7b, 0x75, 0x35, 0x7a, 0x08, 0xdb, 0x23, 0xa0, 0x71, 0x6b, 0xf6, + 0xd0, 0xb8, 0xeb, 0x41, 0x47, 0x8c, 0xba, 0x90, 0xf9, 0xa7, 0x69, 0xe8, 0x4c, 0xbc, 0xab, 0xfe, + 0xd3, 0x7b, 0xea, 0xa9, 0x3f, 0xb5, 0xa7, 0x7e, 0x06, 0xe4, 0x55, 0x1f, 0x31, 0x5c, 0x64, 0x85, + 0xb6, 0xde, 0x35, 0x0a, 0x89, 0xa5, 0x57, 0xfd, 0x1e, 0xcb, 0x9f, 0x19, 0x14, 0x7a, 0x4c, 0xbe, + 0x80, 0xb6, 0xa3, 0x92, 0x85, 0xae, 0xb9, 0x9a, 0x44, 0xc3, 0x32, 0x72, 0xbd, 0x40, 0x98, 0x25, + 0xbb, 0x0f, 0x6d, 0x7f, 0x9b, 0xab, 0xec, 0xd3, 0xb9, 0x30, 0x20, 0x23, 0x22, 0x7c, 0x26, 0x7b, + 0x24, 0xeb, 0x03, 0x5c, 0x13, 0xe8, 0x8e, 0xe7, 0x6d, 0x1c, 0x6c, 0x8c, 0xa8, 0xd8, 0x4e, 0xd0, + 0x9d, 0xed, 0x7f, 0x0a, 0x1b, 0x13, 0x0c, 0x86, 0x51, 0x61, 0x36, 0x8f, 0x36, 0x2a, 0xd6, 0x2f, + 0x9b, 0xdd, 0x33, 0x62, 0xf2, 0x14, 0xee, 0xa6, 0x5c, 0xf0, 0xb4, 0x48, 0xc3, 0x48, 0x8a, 0xf2, + 0xf2, 0x65, 0x44, 0x1b, 0xa3, 0x64, 0x31, 0xd8, 0x71, 0x7a, 0x7b, 0x95, 0x9a, 0x7f, 0x3c, 0xa0, + 0xf0, 0xd4, 0x11, 0x6f, 0x07, 0xdd, 0x1b, 0xf2, 0xa6, 0xb3, 0xbc, 0x61, 0xc0, 0x8b, 0xbf, 0x52, + 0x5c, 0xce, 0xa4, 0x8b, 0x94, 0x7f, 0xbe, 0x01, 0x9d, 0x89, 0x5f, 0x1c, 0x90, 0x27, 0x70, 0x97, + 0x9d, 0x67, 0x2c, 0x32, 0x81, 0xe2, 0x37, 0x65, 0xd6, 0x80, 0x0d, 0x64, 0x1b, 0x2c, 0x77, 0x4a, + 0x3d, 0x9f, 0xc8, 0x18, 0xb2, 0x21, 0xfd, 0x18, 0x96, 0x68, 0x92, 0x9d, 0x50, 0x6f, 0xbd, 0x6f, + 0x12, 0x2d, 0xb7, 0x11, 0x54, 0x2f, 0xee, 0x7b, 0x70, 0x7b, 0xb4, 0x43, 0x69, 0x14, 0x27, 0x8b, + 0x23, 0x8d, 0x89, 0x99, 0xb3, 0x22, 0x1b, 0xe6, 0x34, 0xc6, 0x1b, 0x6a, 0xcd, 0x22, 0xaf, 0x74, + 0xb8, 0xdb, 0x8c, 0x75, 0xa7, 0xd0, 0xab, 0xe4, 0x23, 0x07, 0xc9, 0xff, 0x36, 0x05, 0x9d, 0x89, + 0x9f, 0x4f, 0x90, 0xdf, 0xc0, 0xe6, 0x35, 0xf7, 0x9e, 0xb6, 0xf1, 0xeb, 0x8a, 0xab, 0x2e, 0x3a, + 0x3f, 0x83, 0xad, 0x4b, 0x68, 0x53, 0x9e, 0x4e, 0xb0, 0x09, 0x70, 0x57, 0x38, 0xe3, 0xf0, 0x03, + 0x2e, 0x9e, 0xa2, 0xdc, 0xec, 0x7e, 0x26, 0xdc, 0x60, 0xb6, 0xf0, 0x06, 0x73, 0x65, 0x38, 0x7e, + 0x75, 0x59, 0x76, 0x97, 0x53, 0xd0, 0x99, 0xf8, 0x99, 0x06, 0xf9, 0x08, 0x48, 0x21, 0x34, 0x4f, + 0x5c, 0x5d, 0x70, 0x4e, 0xd8, 0x67, 0x58, 0x46, 0x09, 0x06, 0x91, 0x33, 0xfe, 0x10, 0xb6, 0xca, + 0x8b, 0x3d, 0xef, 0x4b, 0x91, 0xca, 0x8b, 0x1b, 0xe8, 0xc5, 0x86, 0x53, 0xa9, 0x0d, 0x8e, 0x79, + 0xf3, 0xef, 0x37, 0x60, 0xfd, 0x8a, 0xcf, 0x39, 0xc8, 0x5f, 0xc1, 0xfb, 0x82, 0x9d, 0x8d, 0x1c, + 0xb5, 0xe5, 0x6c, 0xc8, 0x95, 0x76, 0xe7, 0xc9, 0x4a, 0xd3, 0x5c, 0x8f, 0xba, 0xf9, 0xae, 0x60, + 0x67, 0x1e, 0x5d, 0xe0, 0xa9, 0x1f, 0x1a, 0x6d, 0xe7, 0xfb, 0x23, 0xb8, 0x83, 0xcf, 0xc8, 0x46, + 0x0f, 0xf2, 0xc6, 0xbd, 0xdf, 0x72, 0x4a, 0xbe, 0x83, 0xa5, 0x0a, 0x46, 0x95, 0x59, 0x19, 0x7c, + 0xbc, 0x79, 0xd8, 0x84, 0x2b, 0x7b, 0xad, 0x36, 0x17, 0xac, 0x17, 0x8a, 0xf9, 0xd8, 0x52, 0x4c, + 0x8e, 0xe1, 0xbd, 0x89, 0xb8, 0x70, 0xc2, 0xfb, 0xb7, 0x01, 0xfa, 0x4e, 0x36, 0x81, 0xe7, 0xe5, + 0xd8, 0x9c, 0xb8, 0x77, 0x1a, 0xc2, 0xc6, 0x95, 0x1f, 0xad, 0x98, 0x80, 0x2d, 0xa7, 0xad, 0xfe, + 0x24, 0xa6, 0x7a, 0xee, 0x29, 0x7b, 0xfb, 0xed, 0x34, 0x2a, 0x96, 0xd1, 0x49, 0x7b, 0xf4, 0xe2, + 0x5f, 0xbf, 0xdf, 0x99, 0xfa, 0xc3, 0xf7, 0x3b, 0x53, 0xff, 0xf5, 0xfd, 0xce, 0xd4, 0xdf, 0xff, + 0xb0, 0xf3, 0xc6, 0x1f, 0x7e, 0xd8, 0x79, 0xe3, 0x8f, 0x3f, 0xec, 0xbc, 0xf1, 0xf5, 0x2f, 0x87, + 0x5c, 0x9f, 0x14, 0xfd, 0x7b, 0x91, 0x4c, 0xef, 0x67, 0xb9, 0x8c, 0x8b, 0x48, 0xab, 0x88, 0x8f, + 0x7d, 0x37, 0xe8, 0x7f, 0x08, 0xa3, 0x2f, 0x32, 0xa6, 0xfa, 0x37, 0xf1, 0x53, 0xbf, 0x9f, 0xff, + 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0xe1, 0x77, 0xba, 0x67, 0x28, 0x00, 0x00, } func (this *Params) Equal(that interface{}) bool { @@ -2328,6 +2337,9 @@ func (this *EpochParams) Equal(that interface{}) bool { if !this.PocSlotAllocation.Equal(that1.PocSlotAllocation) { return false } + if this.ConfirmationPocSafetyWindow != that1.ConfirmationPocSafetyWindow { + return false + } return true } func (this *ValidationParams) Equal(that interface{}) bool { @@ -3352,6 +3364,11 @@ func (m *EpochParams) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ConfirmationPocSafetyWindow != 0 { + i = encodeVarintParams(dAtA, i, uint64(m.ConfirmationPocSafetyWindow)) + i-- + dAtA[i] = 0x78 + } if m.PocSlotAllocation != nil { { size, err := m.PocSlotAllocation.MarshalToSizedBuffer(dAtA[:i]) @@ -4808,6 +4825,9 @@ func (m *EpochParams) Size() (n int) { l = m.PocSlotAllocation.Size() n += 1 + l + sovParams(uint64(l)) } + if m.ConfirmationPocSafetyWindow != 0 { + n += 1 + sovParams(uint64(m.ConfirmationPocSafetyWindow)) + } return n } @@ -6800,6 +6820,25 @@ func (m *EpochParams) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 15: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfirmationPocSafetyWindow", wireType) + } + m.ConfirmationPocSafetyWindow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowParams + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ConfirmationPocSafetyWindow |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipParams(dAtA[iNdEx:]) diff --git a/test-net-cloud/nebius/genesis-overrides.json b/test-net-cloud/nebius/genesis-overrides.json index a988cd4f3..2b5b0ab13 100644 --- a/test-net-cloud/nebius/genesis-overrides.json +++ b/test-net-cloud/nebius/genesis-overrides.json @@ -71,6 +71,7 @@ }, "params": { "epoch_params": { + "confirmation_poc_safety_window": "5", "epoch_length": "360", "epoch_multiplier": "1", "epoch_shift": "300", diff --git a/testermint/src/main/kotlin/data/AppExport.kt b/testermint/src/main/kotlin/data/AppExport.kt index a9a3cd3d6..0cb8f3ae5 100644 --- a/testermint/src/main/kotlin/data/AppExport.kt +++ b/testermint/src/main/kotlin/data/AppExport.kt @@ -144,6 +144,7 @@ data class EpochParams( val pocPruningMax: Long, @SerializedName("poc_slot_allocation") val pocSlotAllocation: Decimal?, + val confirmationPocSafetyWindow: Long, ) data class Decimal( diff --git a/testermint/src/test/kotlin/ConfirmationPoCMultiNodeTests.kt b/testermint/src/test/kotlin/ConfirmationPoCMultiNodeTests.kt index ce9c22fec..b6d138c0e 100644 --- a/testermint/src/test/kotlin/ConfirmationPoCMultiNodeTests.kt +++ b/testermint/src/test/kotlin/ConfirmationPoCMultiNodeTests.kt @@ -377,6 +377,7 @@ fun createConfirmationPoCSpec( this[EpochParams::pocValidationDuration] = 4L this[EpochParams::pocExchangeDuration] = 2L this[EpochParams::pocSlotAllocation] = Decimal.fromDouble(pocSlotAllocation) + this[EpochParams::confirmationPocSafetyWindow] = 0L } this[InferenceParams::confirmationPocParams] = spec { this[ConfirmationPoCParams::expectedConfirmationsPerEpoch] = expectedConfirmationsPerEpoch From a401e2ca9c3a2356ec9a15d578b8f624c22c0c78 Mon Sep 17 00:00:00 2001 From: David and Daniil Liberman Date: Sun, 31 Aug 2025 18:09:09 -0700 Subject: [PATCH 02/31] Described the proposal --- proposals/onboarding-clarity-v1/README.md | 154 ++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 proposals/onboarding-clarity-v1/README.md diff --git a/proposals/onboarding-clarity-v1/README.md b/proposals/onboarding-clarity-v1/README.md new file mode 100644 index 000000000..0000ee3da --- /dev/null +++ b/proposals/onboarding-clarity-v1/README.md @@ -0,0 +1,154 @@ +# Onboarding Clarity Enhancement + +## Overview + +This proposal addresses critical user experience issues during node onboarding by implementing clear state reporting, proactive testing, and intelligent waiting period management. The current onboarding process leaves users confused about system status, with unclear error messages and no guidance on expected waiting periods or next steps. + +## Problem Statement + +### Current Onboarding Issues + +**Unclear Waiting States**: When users install and start nodes, there is no clear indication that nothing will happen until the next Proof-of-Compute (PoC) cycle begins. Users see their nodes running but receive no feedback about the expected waiting period or what will happen next. + +**Confusing Error Messages**: The API node shows "there is no model for ml node" messages even when the participant is not yet active, creating confusion about whether the setup is correct. + +**Lack of Proactive Testing**: New MLnodes are not tested before participating in PoC, leading to failures during critical consensus periods that could have been detected earlier. + +**Poor Restart Handling**: When API nodes restart, they don't properly check if participants are part of the active participant set, leading to inconsistent behavior. + +**Insufficient Status Information**: Users cannot determine: +- How long until the next PoC cycle +- Whether their MLnode is properly configured +- If they can safely turn off servers during waiting periods +- When they should be online for participation + +## Proposed Solution + +**Better Status Messages:** +- When MLnode registered, the latest and most visible log should be "waiting for PoC" +- Tell users exactly when next PoC starts +- Tell users when they should bring the server online, and when they can safely turn off servers temporarily +- Show: Info message "Waiting for next PoC cycle (starts in 2h 15m) - you can safely turn off the server and restart it 10 minutes before PoC" + +**Proactive Testing:** +- When a new MLnode is registered (and there's >1 hour until PoC), automatically test it +- Test: model loading, health check, inference request +- Only show "waiting for PoC" if the test passes +- If test fails, show clear error with specific problem + +**Better Restart Handling:** +- When API node restarts, check if participant is actually in active set +- Don't show confusing error messages if participant isn't active yet +- Instead of: Error "there is no model for ml node" +- Show: Info message "Participant not yet active - model assignment pending (normal for new participants)" + +### Enhanced State Management System + +The proposal introduces a comprehensive state management system that provides clear feedback at every stage of the onboarding and participation lifecycle. + +#### New MLnode States + +**Enhanced State Enumeration**: +- `WAITING_FOR_POC` - MLnode is configured and waiting for next PoC cycle +- `TESTING` - MLnode is undergoing pre-PoC validation testing +- `TEST_FAILED` - MLnode failed validation testing + +**Timing Guidance Through Messages**: +The same `WAITING_FOR_POC` state provides different user messages based on timing: +- "Waiting for next PoC cycle (starts in 2h 15m) - you can safely turn off the server and restart it 10 minutes before PoC" +- "PoC starting soon (in 8 minutes) - MLnode must be online now" + +#### New API Node Participant States + +**Participant Status Tracking**: +- `INACTIVE_WAITING` - Participant registered but not yet in active set +- `ACTIVE_PARTICIPATING` - Participant is in active set and participating in current epoch + +### Proactive MLnode Testing System + +#### Pre-PoC Validation Flow + +**Testing Trigger Conditions**: +- New MLnode registered OR configuration changes detected, when more than 1 hour until next PoC +- Manual testing request through admin interface + +**Testing Process**: +1. **Model Loading Test**: MLnode switches to "testing" state and performs similar operations as in "inference" state - load configured models and verify successful loading +2. **Health Check**: Perform inference health check to ensure model is functional +3. **Response Validation**: Send test inference request and validate response +4. **Performance Baseline**: Record loading time and response time metrics + +**Test Result Actions**: +- **Success**: Switch MLnode to `WAITING_FOR_POC` state +- **Failure**: Switch to `TEST_FAILED` state with detailed error reporting in both MLnode and API node logs + +### Intelligent Timing System + +#### PoC Schedule Awareness + +**Timing Calculations**: +- Calculate time until next PoC cycle using epoch parameters +- Determine safe offline windows (more than 10 minutes until PoC) +- Provide countdown timers for user interfaces +- Alert users when they should be online + +**Existing Epoch Structure Integration**: +- Use existing `Epoch` structure with PoC start block height and upcoming epoch information +- Leverage existing `EpochParams` with `PocStageDuration`, `PocValidationDuration`, and other timing parameters +- Utilize current `chainphase.EpochState` and block height tracking for accurate PoC timing calculations + +### Enhanced Logging and Status Reporting + +#### User-Friendly Status Messages + +**Clear State Communications**: +- "Waiting for next PoC cycle (starts in 2h 15m) - you can safely turn off the server and restart it 10 minutes before PoC" +- "Testing MLnode configuration - model loading in progress" +- "MLnode test failed: model 'Qwen/Qwen2.5-7B-Instruct' could not be loaded" +- "PoC starting soon (in 8 minutes) - MLnode must be online now" + +**Contextual Error Messages**: +- Suppress "no model for ml node" messages when participant is inactive +- Show clear explanations: "Participant not yet active - model assignment will occur after joining active set" +- Provide actionable guidance: "MLnode will be tested automatically when there is more than 1 hour until next PoC" + +#### Enhanced Logging Categories + +**Extend Existing Logging System**: +- Enhance existing `types.Nodes` logging with onboarding state transitions +- Add testing-specific logs within existing `types.Nodes` category +- Use existing `types.Participants` for participant status changes +- Integrate timing guidance into existing log categories rather than creating new ones + +### Implementation Architecture + +#### API Node Enhancements + +**New Components for Admin Server** (`decentralized-api/internal/server/admin/`): +- `OnboardingStateManager` - Centralized state tracking for onboarding process +- `MLnodeTestingOrchestrator` - Manages pre-PoC testing workflows +- `TimingCalculator` - Computes PoC schedules and safe offline windows +- `StatusReporter` - Generates user-friendly status messages + +**MLnode Server** (`decentralized-api/internal/server/mlnode/server.go`): +- No changes needed - existing PoC batch endpoints remain the same + +#### Integration Points + +**Broker Integration**: +- Enhance `NodeState` structure with new onboarding-specific fields +- Modify `RegisterNode` command to trigger testing when appropriate +- Update status query results to include timing information + +**Chain Integration**: +- Query active participant status during startup +- Monitor epoch transitions for timing calculations +- Track participant weight changes for status updates + +### Conclusion + +This proposal addresses critical gaps in the Gonka network's onboarding experience by providing clear state communication, proactive testing, and intelligent timing guidance. The implementation focuses on user experience improvements while maintaining system reliability and security. + +The modular architecture allows for incremental deployment and easy maintenance, while the comprehensive testing approach ensures that configuration issues are caught early rather than during critical consensus periods. This results in a more reliable network and significantly improved user experience for node operators. + +The enhanced status reporting and timing guidance eliminate confusion about waiting periods and provide clear actionable information, transforming the onboarding process from a frustrating guessing game into a transparent, guided experience. From b5ecb766c6521a8f9ed946c2fc5e4caf853a3d30 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 16 Dec 2025 10:30:46 +0800 Subject: [PATCH 03/31] base impl for timing caculator --- .../admin/mlnode_testing_orchestrator.go | 6 + .../server/admin/onboarding_state_manager.go | 5 + .../internal/server/admin/status_reporter.go | 6 + .../server/admin/timing_calculator.go | 109 ++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go create mode 100644 decentralized-api/internal/server/admin/onboarding_state_manager.go create mode 100644 decentralized-api/internal/server/admin/status_reporter.go create mode 100644 decentralized-api/internal/server/admin/timing_calculator.go diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go new file mode 100644 index 000000000..aa1865cb7 --- /dev/null +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -0,0 +1,6 @@ +package admin + +type MLnodeTestingOrchestrator struct{} + +func NewMLnodeTestingOrchestrator() *MLnodeTestingOrchestrator { return &MLnodeTestingOrchestrator{} } + diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go new file mode 100644 index 000000000..a5f47e70f --- /dev/null +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -0,0 +1,5 @@ +package admin + +type OnboardingStateManager struct{} + +func NewOnboardingStateManager() *OnboardingStateManager { return &OnboardingStateManager{} } diff --git a/decentralized-api/internal/server/admin/status_reporter.go b/decentralized-api/internal/server/admin/status_reporter.go new file mode 100644 index 000000000..2187ccfcd --- /dev/null +++ b/decentralized-api/internal/server/admin/status_reporter.go @@ -0,0 +1,6 @@ +package admin + +type StatusReporter struct{} + +func NewStatusReporter() *StatusReporter { return &StatusReporter{} } + diff --git a/decentralized-api/internal/server/admin/timing_calculator.go b/decentralized-api/internal/server/admin/timing_calculator.go new file mode 100644 index 000000000..316e13315 --- /dev/null +++ b/decentralized-api/internal/server/admin/timing_calculator.go @@ -0,0 +1,109 @@ +package admin + +import ( + "decentralized-api/chainphase" + + "github.com/productscience/inference/x/inference/types" +) + +type Window struct { + Start int64 + End int64 + Label string +} + +type TimingResult struct { + BlocksUntilNextPoC int64 + SecondsUntilNextPoC int64 +} + +type Countdown struct { + Phase string + NextPoCSeconds int64 + ShouldBeOnline bool +} + +type TimingCalculator struct{} + +func NewTimingCalculator() *TimingCalculator { return &TimingCalculator{} } + +func (t *TimingCalculator) ComputePoCSchedule(ec *types.EpochContext) []Window { + if ec == nil { + return nil + } + return []Window{ + {Start: ec.StartOfPoC(), End: ec.PoCGenerationWindDown() - 1, Label: string(types.PoCGeneratePhase)}, + {Start: ec.PoCGenerationWindDown(), End: ec.StartOfPoCValidation() - 1, Label: string(types.PoCGenerateWindDownPhase)}, + {Start: ec.StartOfPoCValidation(), End: ec.PoCValidationWindDown() - 1, Label: string(types.PoCValidatePhase)}, + {Start: ec.PoCValidationWindDown(), End: ec.EndOfPoCValidation(), Label: string(types.PoCValidateWindDownPhase)}, + {Start: ec.EndOfPoCValidation() + 1, End: ec.NextPoCStart() - 1, Label: string(types.InferencePhase)}, + } +} + +func (t *TimingCalculator) TimeUntilNextPoC(es *chainphase.EpochState, blockTimeSeconds float64) TimingResult { + if es == nil || !es.IsSynced { + return TimingResult{} + } + ec := es.LatestEpoch + current := es.CurrentBlock.Height + next := ec.NextPoCStart() + blocks := next - current + if blocks < 0 { + blocks = 0 + } + seconds := int64(float64(blocks) * blockTimeSeconds) + return TimingResult{BlocksUntilNextPoC: blocks, SecondsUntilNextPoC: seconds} +} + +func (t *TimingCalculator) SafeOffline(es *chainphase.EpochState, blockTimeSeconds float64, minSeconds int64) bool { + if es == nil || !es.IsSynced { + return false + } + phase := es.CurrentPhase + if phase == types.PoCGeneratePhase || phase == types.PoCGenerateWindDownPhase || phase == types.PoCValidatePhase || phase == types.PoCValidateWindDownPhase { + return false + } + tr := t.TimeUntilNextPoC(es, blockTimeSeconds) + return tr.SecondsUntilNextPoC > minSeconds +} + +func (t *TimingCalculator) ComputeSafeOfflineWindows(es *chainphase.EpochState, blockTimeSeconds float64, minSeconds int64) []Window { + if es == nil || !es.IsSynced { + return nil + } + if es.CurrentPhase != types.InferencePhase { + return []Window{} + } + ec := es.LatestEpoch + start := es.CurrentBlock.Height + end := ec.NextPoCStart() - 1 + if end < start { + return []Window{} + } + seconds := int64(float64(end-start) * blockTimeSeconds) + if seconds <= minSeconds { + return []Window{} + } + return []Window{{Start: start, End: end, Label: "OfflineSafe"}} +} + +func (t *TimingCalculator) OnlineAlert(es *chainphase.EpochState, blockTimeSeconds float64, leadSeconds int64) bool { + if es == nil || !es.IsSynced { + return false + } + phase := es.CurrentPhase + if phase == types.PoCGeneratePhase || phase == types.PoCGenerateWindDownPhase || phase == types.PoCValidatePhase || phase == types.PoCValidateWindDownPhase { + return true + } + tr := t.TimeUntilNextPoC(es, blockTimeSeconds) + return tr.SecondsUntilNextPoC <= leadSeconds +} + +func (t *TimingCalculator) Countdown(es *chainphase.EpochState, blockTimeSeconds float64, leadSeconds int64) Countdown { + if es == nil || !es.IsSynced { + return Countdown{} + } + tr := t.TimeUntilNextPoC(es, blockTimeSeconds) + should := t.OnlineAlert(es, blockTimeSeconds, leadSeconds) + return Countdown{Phase: string(es.CurrentPhase), NextPoCSeconds: tr.SecondsUntilNextPoC, ShouldBeOnline: should} +} From 00763b00da6a89d407fc5dc34ac24fdb94037d79 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 16 Dec 2025 12:36:42 +0800 Subject: [PATCH 04/31] base impl for State Management --- .../server/admin/onboarding_state_manager.go | 94 ++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index a5f47e70f..70dc48eeb 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -1,5 +1,95 @@ package admin -type OnboardingStateManager struct{} +import ( + "decentralized-api/chainphase" +) -func NewOnboardingStateManager() *OnboardingStateManager { return &OnboardingStateManager{} } +type MLNodeOnboardingState string + +const ( + MLNodeState_WAITING_FOR_POC MLNodeOnboardingState = "WAITING_FOR_POC" + MLNodeState_TESTING MLNodeOnboardingState = "TESTING" + MLNodeState_TEST_FAILED MLNodeOnboardingState = "TEST_FAILED" +) + +type ParticipantState string + +const ( + ParticipantState_INACTIVE_WAITING ParticipantState = "INACTIVE_WAITING" + ParticipantState_ACTIVE_PARTICIPATING ParticipantState = "ACTIVE_PARTICIPATING" +) + +type OnboardingStateManager struct { + timing *TimingCalculator + blockTimeSeconds float64 + alertLeadSeconds int64 + safeOfflineMinSec int64 +} + +func NewOnboardingStateManager() *OnboardingStateManager { + return &OnboardingStateManager{ + timing: NewTimingCalculator(), + blockTimeSeconds: 6.0, + alertLeadSeconds: 600, + safeOfflineMinSec: 600, + } +} + +func (m *OnboardingStateManager) MLNodeStatus(es *chainphase.EpochState, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { + if testFailed { + return MLNodeState_TEST_FAILED, "Validation testing failed", true + } + if isTesting { + return MLNodeState_TESTING, "Running pre-PoC validation testing", true + } + c := m.timing.Countdown(es, m.blockTimeSeconds, m.alertLeadSeconds) + if c.ShouldBeOnline { + return MLNodeState_WAITING_FOR_POC, "PoC starting soon (in " + formatShortDuration(c.NextPoCSeconds) + ") - MLnode must be online now", true + } + return MLNodeState_WAITING_FOR_POC, "Waiting for next PoC cycle (starts in " + formatShortDuration(c.NextPoCSeconds) + ") - you can safely turn off the server and restart it 10 minutes before PoC", false +} + +func (m *OnboardingStateManager) ParticipantStatus(isActive bool) ParticipantState { + if isActive { + return ParticipantState_ACTIVE_PARTICIPATING + } + return ParticipantState_INACTIVE_WAITING +} + +func formatShortDuration(seconds int64) string { + if seconds <= 0 { + return "0s" + } + h := seconds / 3600 + m := (seconds % 3600) / 60 + s := seconds % 60 + if h > 0 && m > 0 { + return itoa(h) + "h " + itoa(m) + "m" + } + if h > 0 { + return itoa(h) + "h" + } + if m > 0 && s > 0 { + return itoa(m) + "m " + itoa(s) + "s" + } + if m > 0 { + return itoa(m) + "m" + } + return itoa(s) + "s" +} + +func itoa(v int64) string { + return fmtInt(v) +} + +func fmtInt(v int64) string { + var buf [20]byte + i := len(buf) + n := v + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) +} From 29cb0194d72dd665e433d05318b0d0e1d3b396ae Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 16 Dec 2025 14:38:45 +0800 Subject: [PATCH 05/31] base impl for user-friendly status messages --- .../internal/server/admin/status_reporter.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/decentralized-api/internal/server/admin/status_reporter.go b/decentralized-api/internal/server/admin/status_reporter.go index 2187ccfcd..dfd38672c 100644 --- a/decentralized-api/internal/server/admin/status_reporter.go +++ b/decentralized-api/internal/server/admin/status_reporter.go @@ -1,6 +1,106 @@ package admin +import ( + "decentralized-api/logging" + + "github.com/productscience/inference/x/inference/types" +) + type StatusReporter struct{} func NewStatusReporter() *StatusReporter { return &StatusReporter{} } +func (r *StatusReporter) BuildMLNodeMessage(state MLNodeOnboardingState, secondsUntilNextPoC int64, failingModel string) string { + switch state { + case MLNodeState_TESTING: + return "Testing MLnode configuration - model loading in progress" + case MLNodeState_TEST_FAILED: + if failingModel == "" { + return "MLnode test failed" + } + return "MLnode test failed: model '" + failingModel + "' could not be loaded" + case MLNodeState_WAITING_FOR_POC: + if secondsUntilNextPoC <= 600 { + return "PoC starting soon (in " + formatShortDuration(secondsUntilNextPoC) + ") - MLnode must be online now" + } + return "Waiting for next PoC cycle (starts in " + formatShortDuration(secondsUntilNextPoC) + ") - you can safely turn off the server and restart it 10 minutes before PoC" + default: + return "" + } +} + +func (r *StatusReporter) BuildParticipantMessage(pstate ParticipantState) string { + switch pstate { + case ParticipantState_ACTIVE_PARTICIPATING: + return "Participant is in active set and participating" + case ParticipantState_INACTIVE_WAITING: + return "Participant not yet active - model assignment will occur after joining active set" + default: + return "" + } +} + +func (r *StatusReporter) ShouldSuppressNoModelMessage(participantActive bool) bool { + return !participantActive +} + +func (r *StatusReporter) BuildNoModelGuidance(secondsUntilNextPoC int64) string { + if secondsUntilNextPoC > 3600 { + return "MLnode will be tested automatically when there is more than 1 hour until next PoC" + } + return "" +} + +func (r *StatusReporter) LogOnboardingTransition(prev MLNodeOnboardingState, next MLNodeOnboardingState) { + logging.Info("Onboarding state transition", types.Nodes, "prev", string(prev), "next", string(next)) +} + +func (r *StatusReporter) LogTesting(message string) { + logging.Info(message, types.Nodes) +} + +func (r *StatusReporter) LogParticipantStatusChange(prev ParticipantState, next ParticipantState) { + logging.Info("Participant status change", types.Participants, "prev", string(prev), "next", string(next)) +} + +func (r *StatusReporter) LogTimingGuidance(secondsUntilNextPoC int64) { + logging.Info("Timing guidance", types.Nodes, "seconds_until_next_poc", secondsUntilNextPoC) +} + +func formatShortDuration(seconds int64) string { + if seconds <= 0 { + return "0s" + } + h := seconds / 3600 + m := (seconds % 3600) / 60 + s := seconds % 60 + if h > 0 && m > 0 { + return itoa(h) + "h " + itoa(m) + "m" + } + if h > 0 { + return itoa(h) + "h" + } + if m > 0 && s > 0 { + return itoa(m) + "m " + itoa(s) + "s" + } + if m > 0 { + return itoa(m) + "m" + } + return itoa(s) + "s" +} + +func itoa(v int64) string { + return fmtInt(v) +} + +func fmtInt(v int64) string { + var buf [20]byte + i := len(buf) + n := v + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) +} From aa19222400b84b8871d940c88ae1b18234c95cdd Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 16 Dec 2025 16:04:10 +0800 Subject: [PATCH 06/31] base impl for Proactive MLnode Testing System --- .../admin/mlnode_testing_orchestrator.go | 104 +++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index aa1865cb7..f3ecdcfe3 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -1,6 +1,106 @@ package admin -type MLnodeTestingOrchestrator struct{} +import ( + "context" + "decentralized-api/apiconfig" + "decentralized-api/logging" + "decentralized-api/mlnodeclient" + "time" -func NewMLnodeTestingOrchestrator() *MLnodeTestingOrchestrator { return &MLnodeTestingOrchestrator{} } + "github.com/productscience/inference/x/inference/types" +) +type TestResultStatus string + +const ( + TestSuccess TestResultStatus = "SUCCESS" + TestFailed TestResultStatus = "FAILED" +) + +type TestMetrics struct { + LoadMs map[string]int64 + HealthMs int64 + RespMs int64 +} + +type TestResult struct { + NodeId string + Status TestResultStatus + FailingModel string + Error string + Metrics TestMetrics +} + +type MLnodeTestingOrchestrator struct { + configManager *apiconfig.ConfigManager + blockTimeSeconds float64 +} + +func NewMLnodeTestingOrchestrator(cm *apiconfig.ConfigManager) *MLnodeTestingOrchestrator { + return &MLnodeTestingOrchestrator{configManager: cm, blockTimeSeconds: 6.0} +} + +func (o *MLnodeTestingOrchestrator) ShouldAutoTest(secondsUntilNextPoC int64) bool { + return secondsUntilNextPoC > 3600 +} + +func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apiconfig.InferenceNodeConfig) *TestResult { + version := o.configManager.GetCurrentNodeVersion() + pocUrl := getPoCUrlWithVersion(node, version) + inferenceUrl := formatURL(node.Host, node.InferencePort, node.InferenceSegment) + client := mlnodeclient.NewNodeClient(pocUrl, inferenceUrl) + + metrics := TestMetrics{LoadMs: map[string]int64{}} + + for modelId, cfg := range node.Models { + start := time.Now() + err := client.InferenceUp(ctx, modelId, cfg.Args) + metrics.LoadMs[modelId] = time.Since(start).Milliseconds() + if err != nil { + logging.Error("MLnode test failed during model loading", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) + return &TestResult{NodeId: node.Id, Status: TestFailed, FailingModel: modelId, Error: err.Error(), Metrics: metrics} + } + } + + startHealth := time.Now() + ok, err := client.InferenceHealth(ctx) + metrics.HealthMs = time.Since(startHealth).Milliseconds() + if err != nil || !ok { + if err != nil { + logging.Error("MLnode health check failed", types.Nodes, "node_id", node.Id, "error", err) + return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} + } + logging.Error("MLnode health check not OK", types.Nodes, "node_id", node.Id) + return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "health_not_ok", Metrics: metrics} + } + + metrics.RespMs = 0 + + logging.Info("MLnode test succeeded", types.Nodes, "node_id", node.Id) + return &TestResult{NodeId: node.Id, Status: TestSuccess, Metrics: metrics} +} + +func (o *MLnodeTestingOrchestrator) RunAutoTests(ctx context.Context, secondsUntilNextPoC int64) []TestResult { + if !o.ShouldAutoTest(secondsUntilNextPoC) { + return nil + } + nodes := o.configManager.GetNodes() + results := make([]TestResult, 0, len(nodes)) + for _, n := range nodes { + r := o.RunNodeTest(ctx, n) + if r != nil { + results = append(results, *r) + } + } + return results +} + +func (o *MLnodeTestingOrchestrator) RunManualTest(ctx context.Context, nodeId string) *TestResult { + nodes := o.configManager.GetNodes() + for _, n := range nodes { + if n.Id == nodeId { + return o.RunNodeTest(ctx, n) + } + } + return &TestResult{NodeId: nodeId, Status: TestFailed, Error: "node_not_found"} +} From c93eb3500afee9e0415660b49db1ad9bc100ff6e Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Wed, 17 Dec 2025 18:04:22 +0800 Subject: [PATCH 07/31] refine the output for getNodes --- decentralized-api/broker/broker.go | 14 ++++++++ .../internal/server/admin/node_handlers.go | 33 +++++++++++++++++++ .../server/admin/onboarding_state_manager.go | 13 ++++++++ 3 files changed, 60 insertions(+) diff --git a/decentralized-api/broker/broker.go b/decentralized-api/broker/broker.go index 9b62841e6..d5b3e483f 100644 --- a/decentralized-api/broker/broker.go +++ b/decentralized-api/broker/broker.go @@ -254,6 +254,13 @@ type NodeState struct { // Epoch-specific data, populated from the chain EpochModels map[string]types.Model `json:"epoch_models"` EpochMLNodes map[string]types.MLNodeInfo `json:"epoch_ml_nodes"` + + Timing *TimingInfo `json:"timing,omitempty"` + + UserMessage string `json:"user_message,omitempty"` + Guidance string `json:"guidance,omitempty"` + ParticipantState string `json:"participant_state,omitempty"` + MLNodeOnboardingState string `json:"mlnode_state,omitempty"` } func (s NodeState) MarshalJSON() ([]byte, error) { @@ -349,6 +356,13 @@ type NodeResponse struct { State NodeState `json:"state"` } +type TimingInfo struct { + CurrentPhase string `json:"current_phase"` + BlocksUntilNextPoC int64 `json:"blocks_until_next_poc"` + SecondsUntilNextPoC int64 `json:"seconds_until_next_poc"` + ShouldBeOnline bool `json:"should_be_online"` +} + func NewBroker(chainBridge BrokerChainBridge, phaseTracker *chainphase.ChainPhaseTracker, participantInfo participant.CurrenParticipantInfo, callbackUrl string, clientFactory mlnodeclient.ClientFactory, configManager *apiconfig.ConfigManager) *Broker { broker := &Broker{ highPriorityCommands: make(chan Command, 100), diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 54f3665c8..39864b95b 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -17,6 +17,39 @@ func (s *Server) getNodes(ctx echo.Context) error { logging.Error("Error getting nodes", types.Nodes, "error", err) return err } + osm := NewOnboardingStateManager() + sr := NewStatusReporter() + + for i := range nodes { + state := &nodes[i].State + participantActive := len(state.EpochMLNodes) > 0 + pstate := osm.ParticipantStatus(participantActive) + state.ParticipantState = string(pstate) + + var secs int64 + if state.Timing != nil { + secs = state.Timing.SecondsUntilNextPoC + sr.LogTimingGuidance(secs) + } + + isTesting := false + testFailed := state.FailureReason != "" + mlnodeState, _, _ := osm.MLNodeStatusSimple(secs, isTesting, testFailed) + mlnodeMsg := sr.BuildMLNodeMessage(mlnodeState, secs, "") + + state.MLNodeOnboardingState = string(mlnodeState) + state.UserMessage = mlnodeMsg + + if sr.ShouldSuppressNoModelMessage(participantActive) { + state.Guidance = sr.BuildParticipantMessage(pstate) + } else { + guidance := sr.BuildNoModelGuidance(secs) + if guidance != "" { + state.Guidance = guidance + } + } + } + return ctx.JSON(http.StatusOK, nodes) } diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index 70dc48eeb..a4cc8f537 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -56,6 +56,19 @@ func (m *OnboardingStateManager) ParticipantStatus(isActive bool) ParticipantSta return ParticipantState_INACTIVE_WAITING } +func (m *OnboardingStateManager) MLNodeStatusSimple(secondsUntilNextPoC int64, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { + if testFailed { + return MLNodeState_TEST_FAILED, "Validation testing failed", true + } + if isTesting { + return MLNodeState_TESTING, "Running pre-PoC validation testing", true + } + if secondsUntilNextPoC <= m.alertLeadSeconds { + return MLNodeState_WAITING_FOR_POC, "PoC starting soon (in " + formatShortDuration(secondsUntilNextPoC) + ") - MLnode must be online now", true + } + return MLNodeState_WAITING_FOR_POC, "Waiting for next PoC cycle (starts in " + formatShortDuration(secondsUntilNextPoC) + ") - you can safely turn off the server and restart it 10 minutes before PoC", false +} + func formatShortDuration(seconds int64) string { if seconds <= 0 { return "0s" From 77333bb3f4857c2041867912ce40bd98fe771d21 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Wed, 17 Dec 2025 18:11:59 +0800 Subject: [PATCH 08/31] Integrate timing info in broker GetNodesCommand Add broker command to set node failure reason Trigger auto-tests after node registration in admin Trigger auto-tests on version status endpoint Add manual MLnode testing endpoint in admin --- decentralized-api/broker/broker.go | 4 ++ decentralized-api/broker/commands.go | 62 +++++++++++++++++ .../internal/server/admin/node_handlers.go | 66 +++++++++++++++++++ .../internal/server/admin/server.go | 27 +++++--- .../internal/server/admin/upgrade_handlers.go | 36 ++++++++-- 5 files changed, 183 insertions(+), 12 deletions(-) diff --git a/decentralized-api/broker/broker.go b/decentralized-api/broker/broker.go index d5b3e483f..9307b1229 100644 --- a/decentralized-api/broker/broker.go +++ b/decentralized-api/broker/broker.go @@ -486,6 +486,8 @@ func (b *Broker) executeCommand(command Command) { command.Execute(b) case UpdateNodeResultCommand: command.Execute(b) + case SetNodeFailureReasonCommand: + command.Execute(b) default: logging.Error("Unregistered command type", types.Nodes, "type", reflect.TypeOf(command).String()) } @@ -506,6 +508,8 @@ func (b *Broker) QueueMessage(command Command) error { switch command.(type) { case StartPocCommand, InitValidateCommand, InferenceUpAllCommand, UpdateNodeResultCommand, SetNodesActualStatusCommand, SetNodeAdminStateCommand, RegisterNode, RemoveNode, StartTrainingCommand, LockNodesForTrainingCommand, SyncNodesCommand: b.highPriorityCommands <- command + case SetNodeFailureReasonCommand: + b.highPriorityCommands <- command default: b.lowPriorityCommands <- command } diff --git a/decentralized-api/broker/commands.go b/decentralized-api/broker/commands.go index 33e4ffed7..1e21cca7b 100644 --- a/decentralized-api/broker/commands.go +++ b/decentralized-api/broker/commands.go @@ -50,6 +50,27 @@ func (c GetNodesCommand) Execute(b *Broker) { b.mu.RLock() defer b.mu.RUnlock() + // Precompute timing information from current epoch state + var ( + blocksUntilNextPoC int64 + secondsUntilNextPoC int64 + currentPhase types.EpochPhase + hasEpochInfo bool + ) + epochState := b.phaseTracker.GetCurrentEpochState() + if epochState != nil && epochState.IsSynced { + hasEpochInfo = true + currentPhase = epochState.CurrentPhase + currentHeight := epochState.CurrentBlock.Height + nextPoC := epochState.LatestEpoch.NextPoCStart() + blocksUntilNextPoC = nextPoC - currentHeight + if blocksUntilNextPoC < 0 { + blocksUntilNextPoC = 0 + } + // Use default block time of 6 seconds + secondsUntilNextPoC = int64(float64(blocksUntilNextPoC) * 6.0) + } + nodeResponses := make([]NodeResponse, 0, len(b.nodes)) for _, nodeWithState := range b.nodes { // --- Deep copy Node --- @@ -99,6 +120,15 @@ func (c GetNodesCommand) Execute(b *Broker) { Node: nodeCopy, State: stateCopy, }) + if hasEpochInfo { + shouldOnline := currentPhase == types.PoCGeneratePhase || currentPhase == types.PoCGenerateWindDownPhase || currentPhase == types.PoCValidatePhase || currentPhase == types.PoCValidateWindDownPhase || secondsUntilNextPoC <= 600 + nodeResponses[len(nodeResponses)-1].State.Timing = &TimingInfo{ + CurrentPhase: string(currentPhase), + BlocksUntilNextPoC: blocksUntilNextPoC, + SecondsUntilNextPoC: secondsUntilNextPoC, + ShouldBeOnline: shouldOnline, + } + } } logging.Debug("Got nodes", types.Nodes, "size", len(nodeResponses)) c.Response <- nodeResponses @@ -270,3 +300,35 @@ func (c UpdateNodeResultCommand) Execute(b *Broker) { c.Response <- true } + +// SetNodeFailureReasonCommand sets the FailureReason field on a node state directly +// without requiring an in-flight reconciliation. +type SetNodeFailureReasonCommand struct { + NodeId string + Reason string + Response chan bool +} + +func NewSetNodeFailureReasonCommand(nodeId string, reason string) SetNodeFailureReasonCommand { + return SetNodeFailureReasonCommand{ + NodeId: nodeId, + Reason: reason, + Response: make(chan bool, 2), + } +} + +func (c SetNodeFailureReasonCommand) GetResponseChannelCapacity() int { return cap(c.Response) } + +func (c SetNodeFailureReasonCommand) Execute(b *Broker) { + b.mu.Lock() + defer b.mu.Unlock() + + node, exists := b.nodes[c.NodeId] + if !exists { + logging.Warn("SetNodeFailureReason: node not found", types.Nodes, "node_id", c.NodeId) + c.Response <- false + return + } + node.State.FailureReason = c.Reason + c.Response <- true +} diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 39864b95b..4a806715f 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -1,6 +1,7 @@ package admin import ( + "context" "decentralized-api/apiconfig" "decentralized-api/broker" "decentralized-api/logging" @@ -217,9 +218,74 @@ func (s *Server) addNode(newNode apiconfig.InferenceNodeConfig) (apiconfig.Infer return apiconfig.InferenceNodeConfig{}, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to save node configuration: %v", err)) } + // Auto-test trigger: run pre-PoC validation if timing allows (>1h until next PoC) + // Fetch timing info from broker + getCmd := broker.NewGetNodesCommand() + if err := s.nodeBroker.QueueMessage(getCmd); err == nil { + responses := <-getCmd.Response + var secs int64 + for _, resp := range responses { + if resp.Node.Id == newNode.Id && resp.State.Timing != nil { + secs = resp.State.Timing.SecondsUntilNextPoC + break + } + } + if s.tester.ShouldAutoTest(secs) { + s.statusReporter.LogTesting("Auto-testing MLnode configuration") + result := s.tester.RunNodeTest(context.Background(), *node) + if result != nil { + if result.Status == TestFailed { + cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) + _ = s.nodeBroker.QueueMessage(cmd) + } else { + // Clear any previous failure reason on success + cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, "") + _ = s.nodeBroker.QueueMessage(cmd) + } + s.latestTestResults[newNode.Id] = result + } + } + } + return *node, nil } +// postNodeTest triggers a manual MLnode validation test for a specific node +func (s *Server) postNodeTest(ctx echo.Context) error { + nodeId := ctx.Param("id") + if nodeId == "" { + return echo.NewHTTPError(http.StatusBadRequest, "node id is required") + } + + // Find node config by id + var cfgNode *apiconfig.InferenceNodeConfig + nodes := s.configManager.GetNodes() + for i := range nodes { + if nodes[i].Id == nodeId { + cfgNode = &nodes[i] + break + } + } + if cfgNode == nil { + return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("node not found: %s", nodeId)) + } + + // Run test immediately (manual trigger ignores timing window) + result := s.tester.RunNodeTest(context.Background(), *cfgNode) + if result != nil { + if result.Status == TestFailed { + cmd := broker.NewSetNodeFailureReasonCommand(nodeId, result.Error) + _ = s.nodeBroker.QueueMessage(cmd) + } else { + cmd := broker.NewSetNodeFailureReasonCommand(nodeId, "") + _ = s.nodeBroker.QueueMessage(cmd) + } + s.latestTestResults[nodeId] = result + } + + return ctx.JSON(http.StatusOK, result) +} + // enableNode handles POST /admin/v1/nodes/:id/enable func (s *Server) enableNode(c echo.Context) error { nodeId := c.Param("id") diff --git a/decentralized-api/internal/server/admin/server.go b/decentralized-api/internal/server/admin/server.go index 841605cca..50e53e7f0 100644 --- a/decentralized-api/internal/server/admin/server.go +++ b/decentralized-api/internal/server/admin/server.go @@ -33,6 +33,11 @@ type Server struct { cdc *codec.ProtoCodec blockQueue *pserver.BridgeQueue payloadStorage payloadstorage.PayloadStorage + + onboarding *OnboardingStateManager + statusReporter *StatusReporter + tester *MLnodeTestingOrchestrator + latestTestResults map[string]*TestResult } func NewServer( @@ -47,14 +52,18 @@ func NewServer( e := echo.New() e.HTTPErrorHandler = middleware.TransparentErrorHandler s := &Server{ - e: e, - nodeBroker: nodeBroker, - configManager: configManager, - recorder: recorder, - validator: validator, - cdc: cdc, - blockQueue: blockQueue, - payloadStorage: payloadStorage, + e: e, + nodeBroker: nodeBroker, + configManager: configManager, + recorder: recorder, + validator: validator, + cdc: cdc, + blockQueue: blockQueue, + payloadStorage: payloadStorage, + onboarding: NewOnboardingStateManager(), + statusReporter: NewStatusReporter(), + tester: NewMLnodeTestingOrchestrator(configManager), + latestTestResults: map[string]*TestResult{}, } e.Use(middleware.LoggingMiddleware) @@ -66,6 +75,8 @@ func NewServer( g.PUT("nodes/:id", s.createNewNode) g.GET("nodes/upgrade-status", s.getUpgradeStatus) g.POST("nodes/version-status", s.postVersionStatus) + // Manual MLnode validation test + g.POST("nodes/:id/test", s.postNodeTest) g.GET("nodes", s.getNodes) g.DELETE("nodes/:id", s.deleteNode) g.POST("nodes/:id/enable", s.enableNode) diff --git a/decentralized-api/internal/server/admin/upgrade_handlers.go b/decentralized-api/internal/server/admin/upgrade_handlers.go index 4ef589f1a..76c5d09bb 100644 --- a/decentralized-api/internal/server/admin/upgrade_handlers.go +++ b/decentralized-api/internal/server/admin/upgrade_handlers.go @@ -1,9 +1,11 @@ package admin import ( - "net/http" + "context" + "net/http" - "github.com/labstack/echo/v4" + "decentralized-api/broker" + "github.com/labstack/echo/v4" ) func (s *Server) getUpgradeStatus(c echo.Context) error { @@ -30,6 +32,32 @@ func (s *Server) postVersionStatus(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "Version field is required") } - reports := s.nodeBroker.CheckVersionHealth(req.Version) - return c.JSON(http.StatusOK, reports) + reports := s.nodeBroker.CheckVersionHealth(req.Version) + + // Auto-test trigger on version status check when timing allows + getCmd := broker.NewGetNodesCommand() + if err := s.nodeBroker.QueueMessage(getCmd); err == nil { + responses := <-getCmd.Response + for _, resp := range responses { + var secs int64 + if resp.State.Timing != nil { + secs = resp.State.Timing.SecondsUntilNextPoC + } + if s.tester.ShouldAutoTest(secs) { + result := s.tester.RunNodeTest(context.Background(), resp.Node) + if result != nil { + if result.Status == TestFailed { + cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, result.Error) + _ = s.nodeBroker.QueueMessage(cmd) + } else { + cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, "") + _ = s.nodeBroker.QueueMessage(cmd) + } + s.latestTestResults[resp.Node.Id] = result + } + } + } + } + + return c.JSON(http.StatusOK, reports) } From 230ed6c5eecdabcdfa40647bccd26b9f49550de6 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 18 Dec 2025 11:21:57 +0800 Subject: [PATCH 09/31] move redaclared func & type adaptation --- .../internal/server/admin/status_reporter.go | 40 +-------- .../internal/server/admin/upgrade_handlers.go | 82 +++++++++++-------- 2 files changed, 51 insertions(+), 71 deletions(-) diff --git a/decentralized-api/internal/server/admin/status_reporter.go b/decentralized-api/internal/server/admin/status_reporter.go index dfd38672c..302ebb56d 100644 --- a/decentralized-api/internal/server/admin/status_reporter.go +++ b/decentralized-api/internal/server/admin/status_reporter.go @@ -48,7 +48,7 @@ func (r *StatusReporter) BuildNoModelGuidance(secondsUntilNextPoC int64) string if secondsUntilNextPoC > 3600 { return "MLnode will be tested automatically when there is more than 1 hour until next PoC" } - return "" + return "" //todofhq... } func (r *StatusReporter) LogOnboardingTransition(prev MLNodeOnboardingState, next MLNodeOnboardingState) { @@ -66,41 +66,3 @@ func (r *StatusReporter) LogParticipantStatusChange(prev ParticipantState, next func (r *StatusReporter) LogTimingGuidance(secondsUntilNextPoC int64) { logging.Info("Timing guidance", types.Nodes, "seconds_until_next_poc", secondsUntilNextPoC) } - -func formatShortDuration(seconds int64) string { - if seconds <= 0 { - return "0s" - } - h := seconds / 3600 - m := (seconds % 3600) / 60 - s := seconds % 60 - if h > 0 && m > 0 { - return itoa(h) + "h " + itoa(m) + "m" - } - if h > 0 { - return itoa(h) + "h" - } - if m > 0 && s > 0 { - return itoa(m) + "m " + itoa(s) + "s" - } - if m > 0 { - return itoa(m) + "m" - } - return itoa(s) + "s" -} - -func itoa(v int64) string { - return fmtInt(v) -} - -func fmtInt(v int64) string { - var buf [20]byte - i := len(buf) - n := v - for n > 0 { - i-- - buf[i] = byte('0' + n%10) - n /= 10 - } - return string(buf[i:]) -} diff --git a/decentralized-api/internal/server/admin/upgrade_handlers.go b/decentralized-api/internal/server/admin/upgrade_handlers.go index 76c5d09bb..86b8aa90e 100644 --- a/decentralized-api/internal/server/admin/upgrade_handlers.go +++ b/decentralized-api/internal/server/admin/upgrade_handlers.go @@ -1,11 +1,13 @@ package admin import ( - "context" - "net/http" + "context" + "net/http" - "decentralized-api/broker" - "github.com/labstack/echo/v4" + "decentralized-api/apiconfig" + "decentralized-api/broker" + + "github.com/labstack/echo/v4" ) func (s *Server) getUpgradeStatus(c echo.Context) error { @@ -32,32 +34,48 @@ func (s *Server) postVersionStatus(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "Version field is required") } - reports := s.nodeBroker.CheckVersionHealth(req.Version) - - // Auto-test trigger on version status check when timing allows - getCmd := broker.NewGetNodesCommand() - if err := s.nodeBroker.QueueMessage(getCmd); err == nil { - responses := <-getCmd.Response - for _, resp := range responses { - var secs int64 - if resp.State.Timing != nil { - secs = resp.State.Timing.SecondsUntilNextPoC - } - if s.tester.ShouldAutoTest(secs) { - result := s.tester.RunNodeTest(context.Background(), resp.Node) - if result != nil { - if result.Status == TestFailed { - cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, result.Error) - _ = s.nodeBroker.QueueMessage(cmd) - } else { - cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, "") - _ = s.nodeBroker.QueueMessage(cmd) - } - s.latestTestResults[resp.Node.Id] = result - } - } - } - } - - return c.JSON(http.StatusOK, reports) + reports := s.nodeBroker.CheckVersionHealth(req.Version) + + // Auto-test trigger on version status check when timing allows + getCmd := broker.NewGetNodesCommand() + if err := s.nodeBroker.QueueMessage(getCmd); err == nil { + responses := <-getCmd.Response + for _, resp := range responses { + var secs int64 + if resp.State.Timing != nil { + secs = resp.State.Timing.SecondsUntilNextPoC + } + if s.tester.ShouldAutoTest(secs) { + // Convert broker.Node to apiconfig.InferenceNodeConfig + nodeCfg := apiconfig.InferenceNodeConfig{ + Id: resp.Node.Id, + Host: resp.Node.Host, + InferencePort: resp.Node.InferencePort, + InferenceSegment: resp.Node.InferenceSegment, + PoCPort: resp.Node.PoCPort, + PoCSegment: resp.Node.PoCSegment, + MaxConcurrent: resp.Node.MaxConcurrent, + Hardware: resp.Node.Hardware, + Models: make(map[string]apiconfig.ModelConfig), + } + for mid, margs := range resp.Node.Models { + nodeCfg.Models[mid] = apiconfig.ModelConfig{Args: margs.Args} + } + + result := s.tester.RunNodeTest(context.Background(), nodeCfg) + if result != nil { + if result.Status == TestFailed { + cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, result.Error) + _ = s.nodeBroker.QueueMessage(cmd) + } else { + cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, "") + _ = s.nodeBroker.QueueMessage(cmd) + } + s.latestTestResults[resp.Node.Id] = result + } + } + } + } + + return c.JSON(http.StatusOK, reports) } From 512e0b74b08c5e021eea4b8e8bde6cfced7ef4fa Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 18 Dec 2025 15:41:35 +0800 Subject: [PATCH 10/31] Modify RegisterNode command to trigger auto poc testing --- .../broker/node_admin_commands.go | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index bf03e3f39..76d5c0e32 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -1,6 +1,7 @@ package broker import ( + "context" "decentralized-api/apiconfig" "decentralized-api/logging" "fmt" @@ -169,6 +170,51 @@ func (c RegisterNode) Execute(b *Broker) { logging.Info("RegisterNode. Registered node", types.Nodes, "node", c.Node) c.Response <- NodeCommandResponse{Node: &c.Node, Error: nil} + + // Auto-test: if more than 1 hour until next PoC, perform basic validation + go func(node Node) { + epochState := b.phaseTracker.GetCurrentEpochState() + if epochState == nil || !epochState.IsSynced { + return + } + blocks := epochState.LatestEpoch.NextPoCStart() - epochState.CurrentBlock.Height + if blocks < 0 { + blocks = 0 + } + seconds := int64(float64(blocks) * 6.0) + if seconds <= 3600 { + return + } + + version := b.configManager.GetCurrentNodeVersion() + client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Load each configured model + for modelId, cfg := range node.Models { + if err := client.InferenceUp(ctx, modelId, cfg.Args); err != nil { + logging.Error("RegisterNode. Auto-test model load failed", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + return + } + } + + ok, err := client.InferenceHealth(ctx) + if err != nil || !ok { + reason := "health_not_ok" + if err != nil { + reason = err.Error() + } + logging.Error("RegisterNode. Auto-test health check failed", types.Nodes, "node_id", node.Id, "error", reason) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) + return + } + + // Clear failure reason on success + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) + }(node) } // UpdateNode updates an existing node's configuration while preserving runtime state From 2914221cae8d3f3c8345bb885f037376346b3fb2 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Fri, 19 Dec 2025 16:51:30 +0800 Subject: [PATCH 11/31] Query active participant status during startup Track participant weight changes for status updates --- decentralized-api/broker/broker.go | 68 +++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/decentralized-api/broker/broker.go b/decentralized-api/broker/broker.go index 9307b1229..9545a9279 100644 --- a/decentralized-api/broker/broker.go +++ b/decentralized-api/broker/broker.go @@ -107,22 +107,23 @@ func (b *BrokerChainBridgeImpl) GetParams() (*types.QueryParamsResponse, error) } type Broker struct { - highPriorityCommands chan Command - lowPriorityCommands chan Command - nodes map[string]*NodeWithState - mu sync.RWMutex - curMaxNodesNum atomic.Uint64 - chainBridge BrokerChainBridge - nodeWorkGroup *NodeWorkGroup - phaseTracker *chainphase.ChainPhaseTracker - participantInfo participant.CurrenParticipantInfo - callbackUrl string - mlNodeClientFactory mlnodeclient.ClientFactory - reconcileTrigger chan struct{} - lastEpochIndex uint64 - lastEpochPhase types.EpochPhase - statusQueryTrigger chan statusQuerySignal - configManager *apiconfig.ConfigManager + highPriorityCommands chan Command + lowPriorityCommands chan Command + nodes map[string]*NodeWithState + mu sync.RWMutex + curMaxNodesNum atomic.Uint64 + chainBridge BrokerChainBridge + nodeWorkGroup *NodeWorkGroup + phaseTracker *chainphase.ChainPhaseTracker + participantInfo participant.CurrenParticipantInfo + callbackUrl string + mlNodeClientFactory mlnodeclient.ClientFactory + reconcileTrigger chan struct{} + lastEpochIndex uint64 + lastEpochPhase types.EpochPhase + lastParticipantWeight int64 + statusQueryTrigger chan statusQuerySignal + configManager *apiconfig.ConfigManager } // GetParticipantAddress returns the current participant's address if available. @@ -261,6 +262,7 @@ type NodeState struct { Guidance string `json:"guidance,omitempty"` ParticipantState string `json:"participant_state,omitempty"` MLNodeOnboardingState string `json:"mlnode_state,omitempty"` + ParticipantWeight int64 `json:"participant_weight,omitempty"` } func (s NodeState) MarshalJSON() ([]byte, error) { @@ -387,6 +389,18 @@ func NewBroker(chainBridge BrokerChainBridge, phaseTracker *chainphase.ChainPhas // go nodeReconciliationWorker(broker) go nodeStatusQueryWorker(broker) go broker.reconcilerLoop() + + // Startup: try to populate epoch data once chain is synced to expose participant status early + go func() { + for i := 0; i < 10; i++ { + es := broker.phaseTracker.GetCurrentEpochState() + if es != nil && es.IsSynced { + _ = broker.UpdateNodeWithEpochData(es) + return + } + time.Sleep(1 * time.Second) + } + }() return broker } @@ -1540,6 +1554,9 @@ func (b *Broker) UpdateNodeWithEpochData(epochState *chainphase.EpochState) erro parentEpochData := parentGroupResp.GetEpochGroupData() + // Calculate current participant weight by scanning validation weights across subgroups + currentWeight := int64(0) + b.clearNodeEpochData() // 2. Track which nodes are found in epoch data @@ -1567,6 +1584,12 @@ func (b *Broker) UpdateNodeWithEpochData(epochState *chainphase.EpochState) erro for _, weightInfo := range subgroup.ValidationWeights { // Check if the participant is the one this broker is managing if weightInfo.MemberAddress == b.participantInfo.GetAddress() { + // Track participant weight (use ConfirmationWeight if present, else Weight) + w := weightInfo.ConfirmationWeight + if w == 0 { + w = weightInfo.Weight + } + currentWeight += w // 5. Iterate through the ML nodes for this participant in the epoch data b.UpdateNodeEpochData(weightInfo.MlNodes, modelId, *subgroup.ModelSnapshot) // Mark these nodes as found in epoch @@ -1577,6 +1600,19 @@ func (b *Broker) UpdateNodeWithEpochData(epochState *chainphase.EpochState) erro } } + // If participant weight changed, log and update cached weight + if currentWeight != b.lastParticipantWeight { + logging.Info("Participant weight changed", types.Participants, "old", b.lastParticipantWeight, "new", currentWeight, "epoch", epochState.LatestEpoch.EpochIndex) + b.lastParticipantWeight = currentWeight + } + + // Store participant weight on each node state for visibility in admin APIs + b.mu.Lock() + for _, node := range b.nodes { + node.State.ParticipantWeight = currentWeight + } + b.mu.Unlock() + // 6. Populate governance models for nodes not in epoch data (disabled nodes) b.mu.RLock() nodeIds := make([]string, 0, len(b.nodes)) From 0413eb4a74c3952b1d497480a478a100718b3a1a Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Fri, 19 Dec 2025 20:51:04 +0800 Subject: [PATCH 12/31] add test to mock intergrated --- .../internal/server/admin/server_test.go | 144 +++++++++++++++++- .../server/admin/setup_report_test.go | 10 +- 2 files changed, 144 insertions(+), 10 deletions(-) diff --git a/decentralized-api/internal/server/admin/server_test.go b/decentralized-api/internal/server/admin/server_test.go index 61bbe2d07..9e40173e9 100644 --- a/decentralized-api/internal/server/admin/server_test.go +++ b/decentralized-api/internal/server/admin/server_test.go @@ -60,10 +60,9 @@ func (m *mockInferenceQueryClient) EpochGroupData(ctx context.Context, in *types return args.Get(0).(*types.QueryGetEpochGroupDataResponse), args.Error(1) } -func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodeclient.MockClientFactory) { +func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodeclient.MockClientFactory, *chainphase.ChainPhaseTracker) { // Disable model enforcement in tests os.Setenv("ENFORCED_MODEL_ID", "disabled") - // 1. Config Manager tmpFile, err := os.CreateTemp("", "config-*.yaml") assert.NoError(t, err) @@ -108,6 +107,18 @@ func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodecl EpochIndex: 100, ModelId: "", }).Return(parentGroupResp, nil) + // Also mock for next epoch (index 101) + parentGroupRespNext := &types.QueryGetEpochGroupDataResponse{ + EpochGroupData: types.EpochGroupData{ + PocStartBlockHeight: 200, + EpochIndex: 101, + SubGroupModels: []string{"test-model"}, + }, + } + mockQueryClient.On("EpochGroupData", mock.Anything, &types.QueryGetEpochGroupDataRequest{ + EpochIndex: 101, + ModelId: "", + }).Return(parentGroupRespNext, nil) // Mock epoch group data for specific model modelEpochData := &types.QueryGetEpochGroupDataResponse{ @@ -115,12 +126,44 @@ func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodecl PocStartBlockHeight: 100, EpochIndex: 100, ModelSnapshot: &types.Model{Id: "test-model"}, + ValidationWeights: []*types.ValidationWeight{ + { + MemberAddress: "test-participant", + ConfirmationWeight: 123, + Weight: 123, + MlNodes: []*types.MLNodeInfo{ + {NodeId: "node-1"}, + }, + }, + }, }, } mockQueryClient.On("EpochGroupData", mock.Anything, &types.QueryGetEpochGroupDataRequest{ EpochIndex: 100, ModelId: "test-model", }).Return(modelEpochData, nil) + // Mock for next epoch (index 101) + modelEpochDataNext := &types.QueryGetEpochGroupDataResponse{ + EpochGroupData: types.EpochGroupData{ + PocStartBlockHeight: 200, + EpochIndex: 101, + ModelSnapshot: &types.Model{Id: "test-model"}, + ValidationWeights: []*types.ValidationWeight{ + { + MemberAddress: "test-participant", + ConfirmationWeight: 123, + Weight: 123, + MlNodes: []*types.MLNodeInfo{ + {NodeId: "node-1"}, + }, + }, + }, + }, + } + mockQueryClient.On("EpochGroupData", mock.Anything, &types.QueryGetEpochGroupDataRequest{ + EpochIndex: 101, + ModelId: "test-model", + }).Return(modelEpochDataNext, nil) // 3. PhaseTracker phaseTracker := chainphase.NewChainPhaseTracker() @@ -138,11 +181,11 @@ func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodecl // 5. Server s := NewServer(mockCosmos, nodeBroker, configManager, nil, nil, nil) - return s, configManager, mockClientFactory + return s, configManager, mockClientFactory, phaseTracker } func TestGetUpgradeStatus(t *testing.T) { - s, configManager, _ := setupTestServer(t) + s, configManager, _, _ := setupTestServer(t) t.Run("no upgrade plan", func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/admin/v1/nodes/upgrade-status", nil) @@ -168,7 +211,7 @@ func TestGetUpgradeStatus(t *testing.T) { } func TestPostVersionStatus(t *testing.T) { - s, configManager, mockClientFactory := setupTestServer(t) + s, configManager, mockClientFactory, _ := setupTestServer(t) nodeConfig := apiconfig.InferenceNodeConfig{ Id: "node-1", @@ -226,3 +269,94 @@ func TestPostVersionStatus(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rec.Code) }) } + +func TestAdminGetNodesIncludesOnboardingFields(t *testing.T) { + s, configManager, _, phaseTracker := setupTestServer(t) + + nodeConfig := apiconfig.InferenceNodeConfig{ + Id: "node-1", + Host: "localhost", + InferencePort: 8080, + InferenceSegment: "/api/v1", + PoCPort: 8081, + PoCSegment: "/api/v1", + MaxConcurrent: 3, + Models: map[string]apiconfig.ModelConfig{ + "test-model": {Args: []string{}}, + }, + } + + nodes := configManager.GetNodes() + nodes = append(nodes, nodeConfig) + err := configManager.SetNodes(nodes) + assert.NoError(t, err) + + respChan := s.nodeBroker.LoadNodeToBroker(&nodeConfig) + select { + case response := <-respChan: + if response.Error != nil || response.Node == nil { + t.Fatal("failed to register node - node validation failed") + } + case <-time.After(1 * time.Second): + t.Fatal("timed out waiting for node to register") + } + + // Ensure epoch data is applied after node registration + es := phaseTracker.GetCurrentEpochState() + assert.NotNil(t, es) + err = s.nodeBroker.UpdateNodeWithEpochData(es) + assert.NoError(t, err) + + // Advance to next epoch to force an epoch change and ensure weight propagation + phaseTracker.Update( + chainphase.BlockInfo{Height: 200, Hash: "hash-200"}, + &types.Epoch{Index: 101, PocStartBlockHeight: 200}, + &types.EpochParams{}, + true, + nil, + ) + es = phaseTracker.GetCurrentEpochState() + assert.NotNil(t, es) + err = s.nodeBroker.UpdateNodeWithEpochData(es) + assert.NoError(t, err) + + req := httptest.NewRequest(http.MethodGet, "/admin/v1/nodes", nil) + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) + + var responses []map[string]interface{} + dec := json.NewDecoder(bytes.NewReader(rec.Body.Bytes())) + err = dec.Decode(&responses) + assert.NoError(t, err) + + // Find node-1 and assert fields + var found map[string]interface{} + for i := range responses { + nodeObj, _ := responses[i]["node"].(map[string]interface{}) + if nodeObj != nil && nodeObj["id"] == "node-1" { + found = responses[i] + break + } + } + if found == nil { + t.Fatal("node-1 not found in admin response") + } + + stateObj, _ := found["state"].(map[string]interface{}) + if stateObj == nil { + t.Fatal("state missing for node-1") + } + // participant_state should be set and indicate active participation + assert.Equal(t, "ACTIVE_PARTICIPATING", stateObj["participant_state"]) + // mlnode_state should be present (waiting for PoC given our timing) + assert.Equal(t, "WAITING_FOR_POC", stateObj["mlnode_state"]) + // participant_weight should reflect mocked weight + assert.Equal(t, float64(123), stateObj["participant_weight"]) // JSON numbers decode to float64 + // timing should be present with a positive countdown + timingObj, _ := stateObj["timing"].(map[string]interface{}) + if assert.NotNil(t, timingObj) { + secs, _ := timingObj["seconds_until_next_poc"].(float64) + assert.GreaterOrEqual(t, int64(secs), int64(0)) + } +} diff --git a/decentralized-api/internal/server/admin/setup_report_test.go b/decentralized-api/internal/server/admin/setup_report_test.go index 8c0de7c9e..1e90c7892 100644 --- a/decentralized-api/internal/server/admin/setup_report_test.go +++ b/decentralized-api/internal/server/admin/setup_report_test.go @@ -16,7 +16,7 @@ import ( // Test Summary Generation func TestGenerateSummary_AllPass(t *testing.T) { - s, _, _ := setupTestServer(t) + s, _, _, _ := setupTestServer(t) report := &SetupReport{ Checks: []Check{ @@ -38,7 +38,7 @@ func TestGenerateSummary_AllPass(t *testing.T) { } func TestGenerateSummary_WithFailures(t *testing.T) { - s, _, _ := setupTestServer(t) + s, _, _, _ := setupTestServer(t) report := &SetupReport{ Checks: []Check{ @@ -62,7 +62,7 @@ func TestGenerateSummary_WithFailures(t *testing.T) { } func TestGenerateSummary_WithUnavailable(t *testing.T) { - s, _, _ := setupTestServer(t) + s, _, _, _ := setupTestServer(t) report := &SetupReport{ Checks: []Check{ @@ -84,7 +84,7 @@ func TestGenerateSummary_WithUnavailable(t *testing.T) { } func TestGenerateSummary_MLNodeWithNoGPUs(t *testing.T) { - s, _, _ := setupTestServer(t) + s, _, _, _ := setupTestServer(t) report := &SetupReport{ Checks: []Check{ @@ -112,7 +112,7 @@ func TestGenerateSummary_MLNodeWithNoGPUs(t *testing.T) { } func TestGenerateSummary_MLNodeWithUnavailableGPU(t *testing.T) { - s, _, _ := setupTestServer(t) + s, _, _, _ := setupTestServer(t) report := &SetupReport{ Checks: []Check{ From 957d0ffd3341cbda7cf9f4e0e9a0de53b3ac1307 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 23 Dec 2025 00:19:12 +0800 Subject: [PATCH 13/31] logical change for status message dont need to do auto test for upgrade --- .../internal/server/admin/node_handlers.go | 21 ++++----- .../internal/server/admin/status_reporter.go | 6 +-- .../internal/server/admin/upgrade_handlers.go | 46 ------------------- 3 files changed, 11 insertions(+), 62 deletions(-) diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 4a806715f..abce63589 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -35,20 +35,19 @@ func (s *Server) getNodes(ctx echo.Context) error { isTesting := false testFailed := state.FailureReason != "" + if !participantActive { + isTesting = false + testFailed = false + } mlnodeState, _, _ := osm.MLNodeStatusSimple(secs, isTesting, testFailed) - mlnodeMsg := sr.BuildMLNodeMessage(mlnodeState, secs, "") + var userMsg string + if participantActive { + userMsg = sr.BuildMLNodeMessage(mlnodeState, secs, "") + } state.MLNodeOnboardingState = string(mlnodeState) - state.UserMessage = mlnodeMsg - - if sr.ShouldSuppressNoModelMessage(participantActive) { - state.Guidance = sr.BuildParticipantMessage(pstate) - } else { - guidance := sr.BuildNoModelGuidance(secs) - if guidance != "" { - state.Guidance = guidance - } - } + state.UserMessage = userMsg + state.Guidance = sr.BuildParticipantMessage(pstate) } return ctx.JSON(http.StatusOK, nodes) diff --git a/decentralized-api/internal/server/admin/status_reporter.go b/decentralized-api/internal/server/admin/status_reporter.go index 302ebb56d..a49963cd4 100644 --- a/decentralized-api/internal/server/admin/status_reporter.go +++ b/decentralized-api/internal/server/admin/status_reporter.go @@ -40,15 +40,11 @@ func (r *StatusReporter) BuildParticipantMessage(pstate ParticipantState) string } } -func (r *StatusReporter) ShouldSuppressNoModelMessage(participantActive bool) bool { - return !participantActive -} - func (r *StatusReporter) BuildNoModelGuidance(secondsUntilNextPoC int64) string { if secondsUntilNextPoC > 3600 { return "MLnode will be tested automatically when there is more than 1 hour until next PoC" } - return "" //todofhq... + return "" } func (r *StatusReporter) LogOnboardingTransition(prev MLNodeOnboardingState, next MLNodeOnboardingState) { diff --git a/decentralized-api/internal/server/admin/upgrade_handlers.go b/decentralized-api/internal/server/admin/upgrade_handlers.go index 86b8aa90e..4ef589f1a 100644 --- a/decentralized-api/internal/server/admin/upgrade_handlers.go +++ b/decentralized-api/internal/server/admin/upgrade_handlers.go @@ -1,12 +1,8 @@ package admin import ( - "context" "net/http" - "decentralized-api/apiconfig" - "decentralized-api/broker" - "github.com/labstack/echo/v4" ) @@ -35,47 +31,5 @@ func (s *Server) postVersionStatus(c echo.Context) error { } reports := s.nodeBroker.CheckVersionHealth(req.Version) - - // Auto-test trigger on version status check when timing allows - getCmd := broker.NewGetNodesCommand() - if err := s.nodeBroker.QueueMessage(getCmd); err == nil { - responses := <-getCmd.Response - for _, resp := range responses { - var secs int64 - if resp.State.Timing != nil { - secs = resp.State.Timing.SecondsUntilNextPoC - } - if s.tester.ShouldAutoTest(secs) { - // Convert broker.Node to apiconfig.InferenceNodeConfig - nodeCfg := apiconfig.InferenceNodeConfig{ - Id: resp.Node.Id, - Host: resp.Node.Host, - InferencePort: resp.Node.InferencePort, - InferenceSegment: resp.Node.InferenceSegment, - PoCPort: resp.Node.PoCPort, - PoCSegment: resp.Node.PoCSegment, - MaxConcurrent: resp.Node.MaxConcurrent, - Hardware: resp.Node.Hardware, - Models: make(map[string]apiconfig.ModelConfig), - } - for mid, margs := range resp.Node.Models { - nodeCfg.Models[mid] = apiconfig.ModelConfig{Args: margs.Args} - } - - result := s.tester.RunNodeTest(context.Background(), nodeCfg) - if result != nil { - if result.Status == TestFailed { - cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, result.Error) - _ = s.nodeBroker.QueueMessage(cmd) - } else { - cmd := broker.NewSetNodeFailureReasonCommand(resp.Node.Id, "") - _ = s.nodeBroker.QueueMessage(cmd) - } - s.latestTestResults[resp.Node.Id] = result - } - } - } - } - return c.JSON(http.StatusOK, reports) } From 3f2271077a10d9bcfa652494b6e17b0507b96788 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 23 Dec 2025 01:06:26 +0800 Subject: [PATCH 14/31] add auto test when update node --- .../broker/node_admin_commands.go | 93 ++++++++++--------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index 76d5c0e32..b25217513 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -172,49 +172,7 @@ func (c RegisterNode) Execute(b *Broker) { c.Response <- NodeCommandResponse{Node: &c.Node, Error: nil} // Auto-test: if more than 1 hour until next PoC, perform basic validation - go func(node Node) { - epochState := b.phaseTracker.GetCurrentEpochState() - if epochState == nil || !epochState.IsSynced { - return - } - blocks := epochState.LatestEpoch.NextPoCStart() - epochState.CurrentBlock.Height - if blocks < 0 { - blocks = 0 - } - seconds := int64(float64(blocks) * 6.0) - if seconds <= 3600 { - return - } - - version := b.configManager.GetCurrentNodeVersion() - client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // Load each configured model - for modelId, cfg := range node.Models { - if err := client.InferenceUp(ctx, modelId, cfg.Args); err != nil { - logging.Error("RegisterNode. Auto-test model load failed", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) - return - } - } - - ok, err := client.InferenceHealth(ctx) - if err != nil || !ok { - reason := "health_not_ok" - if err != nil { - reason = err.Error() - } - logging.Error("RegisterNode. Auto-test health check failed", types.Nodes, "node_id", node.Id, "error", reason) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) - return - } - - // Clear failure reason on success - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) - }(node) + go b.autoTestNodeIfTimeAllows(node, "RegisterNode") } // UpdateNode updates an existing node's configuration while preserving runtime state @@ -312,6 +270,9 @@ func (c UpdateNode) Execute(b *Broker) { logging.Info("UpdateNode. Updated node configuration", types.Nodes, "node_id", c.Node.Id) c.Response <- NodeCommandResponse{Node: &c.Node, Error: nil} + + // Auto-test: if more than 1 hour until next PoC, perform basic validation after config changes + go b.autoTestNodeIfTimeAllows(updated, "UpdateNode") } type RemoveNode struct { @@ -416,3 +377,49 @@ func (c UpdateNodeHardwareCommand) Execute(b *Broker) { logging.Info("Updated node hardware", types.Nodes, "node_id", c.NodeId, "hardware_count", len(c.Hardware)) c.Response <- nil } + +// autoTestNodeIfTimeAllows performs a basic validation test for a node when +// there is more than 1 hour remaining until the next PoC. It loads each +// configured model and checks inference health, updating the node failure +// reason accordingly. The caller string is used for logging context. +func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { + epochState := b.phaseTracker.GetCurrentEpochState() + if epochState == nil || !epochState.IsSynced { + return + } + blocks := epochState.LatestEpoch.NextPoCStart() - epochState.CurrentBlock.Height + if blocks < 0 { + blocks = 0 + } + seconds := int64(float64(blocks) * 6.0) + if seconds <= 3600 { + return + } + + version := b.configManager.GetCurrentNodeVersion() + client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + for modelId, cfg := range node.Models { + if err := client.InferenceUp(ctx, modelId, cfg.Args); err != nil { + logging.Error(caller+". Auto-test model load failed", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + return + } + } + + ok, err := client.InferenceHealth(ctx) + if err != nil || !ok { + reason := "health_not_ok" + if err != nil { + reason = err.Error() + } + logging.Error(caller+". Auto-test health check failed", types.Nodes, "node_id", node.Id, "error", reason) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) + return + } + + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) +} From 3acbcc9f5c7b860370fc0dc1f013052c3a3fd6e4 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 23 Dec 2025 01:28:24 +0800 Subject: [PATCH 15/31] print getNodes info in api service --- decentralized-api/internal/server/admin/node_handlers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index abce63589..3ccec72da 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -48,6 +48,13 @@ func (s *Server) getNodes(ctx echo.Context) error { state.MLNodeOnboardingState = string(mlnodeState) state.UserMessage = userMsg state.Guidance = sr.BuildParticipantMessage(pstate) + + logging.Info("Admin getNodes state", types.Nodes, + "node_id", nodes[i].Node.Id, + "participant_state", state.ParticipantState, + "mlnode_state", state.MLNodeOnboardingState, + "user_message", state.UserMessage, + "guidance", state.Guidance) } return ctx.JSON(http.StatusOK, nodes) From 40d770eab89bcca4d6a476fa78b2491687051ddf Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Tue, 23 Dec 2025 10:05:09 +0800 Subject: [PATCH 16/31] add log for auto test --- decentralized-api/broker/node_admin_commands.go | 5 +++++ decentralized-api/internal/server/admin/node_handlers.go | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index b25217513..44d2dbc08 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -385,6 +385,7 @@ func (c UpdateNodeHardwareCommand) Execute(b *Broker) { func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { epochState := b.phaseTracker.GetCurrentEpochState() if epochState == nil || !epochState.IsSynced { + logging.Info(caller+". Auto-test skipped: epoch state unavailable", types.Nodes, "node_id", node.Id) return } blocks := epochState.LatestEpoch.NextPoCStart() - epochState.CurrentBlock.Height @@ -392,12 +393,15 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { blocks = 0 } seconds := int64(float64(blocks) * 6.0) + logging.Info(caller+". Auto-test timing evaluated", types.Nodes, "node_id", node.Id, "seconds_until_next_poc", seconds) if seconds <= 3600 { + logging.Info(caller+". Auto-test skipped: insufficient time", types.Nodes, "node_id", node.Id, "seconds_until_next_poc", seconds) return } version := b.configManager.GetCurrentNodeVersion() client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) + logging.Info(caller+". Auto-test starting", types.Nodes, "node_id", node.Id, "models", len(node.Models)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -422,4 +426,5 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { } _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) + logging.Info(caller+". Auto-test passed", types.Nodes, "node_id", node.Id) } diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 3ccec72da..673915338 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -276,7 +276,7 @@ func (s *Server) postNodeTest(ctx echo.Context) error { return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("node not found: %s", nodeId)) } - // Run test immediately (manual trigger ignores timing window) + logging.Info("Admin manual test start", types.Nodes, "node_id", nodeId) result := s.tester.RunNodeTest(context.Background(), *cfgNode) if result != nil { if result.Status == TestFailed { @@ -286,6 +286,7 @@ func (s *Server) postNodeTest(ctx echo.Context) error { cmd := broker.NewSetNodeFailureReasonCommand(nodeId, "") _ = s.nodeBroker.QueueMessage(cmd) } + logging.Info("Admin manual test result", types.Nodes, "node_id", nodeId, "status", string(result.Status), "error", result.Error) s.latestTestResults[nodeId] = result } From db276c49dc6481fbab56e2a7a6ad6f4532f49a2f Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Wed, 24 Dec 2025 15:49:58 +0800 Subject: [PATCH 17/31] Implement chain-based participant active check. minor modify for common func --- decentralized-api/broker/broker.go | 30 +++++++++++++++++ .../internal/server/admin/node_handlers.go | 11 +++++-- .../server/admin/onboarding_state_manager.go | 32 ++++++++----------- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/decentralized-api/broker/broker.go b/decentralized-api/broker/broker.go index 9545a9279..896dd20c6 100644 --- a/decentralized-api/broker/broker.go +++ b/decentralized-api/broker/broker.go @@ -1846,3 +1846,33 @@ func (b *Broker) MergeModelArgs(epochArgs []string, localArgs []string) []string return mergedArgs } + +func (b *Broker) IsParticipantActiveOnChain() (bool, error) { + resp, err := b.chainBridge.GetCurrentEpochGroupData() + if err != nil { + return false, err + } + if resp == nil { + return false, nil + } + epochIndex := resp.EpochGroupData.EpochIndex + subModels := resp.EpochGroupData.SubGroupModels + addr := b.participantInfo.GetAddress() + for _, mid := range subModels { + subgroupResp, err := b.chainBridge.GetEpochGroupDataByModelId(epochIndex, mid) + if err != nil { + continue + } + if subgroupResp == nil { + continue + } + for _, w := range subgroupResp.EpochGroupData.ValidationWeights { + if w.MemberAddress == addr { + if len(w.MlNodes) > 0 { + return true, nil + } + } + } + } + return false, nil +} diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 673915338..6df1d6363 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -20,10 +20,17 @@ func (s *Server) getNodes(ctx echo.Context) error { } osm := NewOnboardingStateManager() sr := NewStatusReporter() + chainActive := false + if s.nodeBroker != nil { + active, err := s.nodeBroker.IsParticipantActiveOnChain() + if err == nil { + chainActive = active + } + } for i := range nodes { state := &nodes[i].State - participantActive := len(state.EpochMLNodes) > 0 + participantActive := chainActive || len(state.EpochMLNodes) > 0 pstate := osm.ParticipantStatus(participantActive) state.ParticipantState = string(pstate) @@ -39,7 +46,7 @@ func (s *Server) getNodes(ctx echo.Context) error { isTesting = false testFailed = false } - mlnodeState, _, _ := osm.MLNodeStatusSimple(secs, isTesting, testFailed) + mlnodeState, _, _ := osm.MLNodeStatus(secs, isTesting, testFailed) var userMsg string if participantActive { userMsg = sr.BuildMLNodeMessage(mlnodeState, secs, "") diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index a4cc8f537..069eb0808 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -1,9 +1,5 @@ package admin -import ( - "decentralized-api/chainphase" -) - type MLNodeOnboardingState string const ( @@ -35,19 +31,19 @@ func NewOnboardingStateManager() *OnboardingStateManager { } } -func (m *OnboardingStateManager) MLNodeStatus(es *chainphase.EpochState, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { - if testFailed { - return MLNodeState_TEST_FAILED, "Validation testing failed", true - } - if isTesting { - return MLNodeState_TESTING, "Running pre-PoC validation testing", true - } - c := m.timing.Countdown(es, m.blockTimeSeconds, m.alertLeadSeconds) - if c.ShouldBeOnline { - return MLNodeState_WAITING_FOR_POC, "PoC starting soon (in " + formatShortDuration(c.NextPoCSeconds) + ") - MLnode must be online now", true - } - return MLNodeState_WAITING_FOR_POC, "Waiting for next PoC cycle (starts in " + formatShortDuration(c.NextPoCSeconds) + ") - you can safely turn off the server and restart it 10 minutes before PoC", false -} +// func (m *OnboardingStateManager) MLNodeStatus(es *chainphase.EpochState, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { +// if testFailed { +// return MLNodeState_TEST_FAILED, "Validation testing failed", true +// } +// if isTesting { +// return MLNodeState_TESTING, "Running pre-PoC validation testing", true +// } +// c := m.timing.Countdown(es, m.blockTimeSeconds, m.alertLeadSeconds) +// if c.ShouldBeOnline { +// return MLNodeState_WAITING_FOR_POC, "PoC starting soon (in " + formatShortDuration(c.NextPoCSeconds) + ") - MLnode must be online now", true +// } +// return MLNodeState_WAITING_FOR_POC, "Waiting for next PoC cycle (starts in " + formatShortDuration(c.NextPoCSeconds) + ") - you can safely turn off the server and restart it 10 minutes before PoC", false +// } func (m *OnboardingStateManager) ParticipantStatus(isActive bool) ParticipantState { if isActive { @@ -56,7 +52,7 @@ func (m *OnboardingStateManager) ParticipantStatus(isActive bool) ParticipantSta return ParticipantState_INACTIVE_WAITING } -func (m *OnboardingStateManager) MLNodeStatusSimple(secondsUntilNextPoC int64, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { +func (m *OnboardingStateManager) MLNodeStatus(secondsUntilNextPoC int64, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { if testFailed { return MLNodeState_TEST_FAILED, "Validation testing failed", true } From ec9069a826e5618ff411c1ac0949ebec38d2354a Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Wed, 24 Dec 2025 21:30:21 +0800 Subject: [PATCH 18/31] send test inference request and validate response is added --- .../broker/node_admin_commands.go | 42 +++++++++++++++++++ .../admin/mlnode_testing_orchestrator.go | 38 ++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index 44d2dbc08..e36b24569 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -1,10 +1,13 @@ package broker import ( + "bytes" "context" "decentralized-api/apiconfig" "decentralized-api/logging" + "encoding/json" "fmt" + "net/http" "strings" "time" @@ -378,6 +381,13 @@ func (c UpdateNodeHardwareCommand) Execute(b *Broker) { c.Response <- nil } +func getFirstModelIdFromNode(node Node) string { + for modelId := range node.Models { + return modelId + } + return "" // Return empty if no models +} + // autoTestNodeIfTimeAllows performs a basic validation test for a node when // there is more than 1 hour remaining until the next PoC. It loads each // configured model and checks inference health, updating the node failure @@ -425,6 +435,38 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { return } + // Perform test inference request to validate response + firstModelId := getFirstModelIdFromNode(node) + if firstModelId != "" { + testRequest := map[string]interface{}{ + "model": firstModelId, + "messages": []map[string]string{{"role": "user", "content": "Hello, how are you?"}}, + "max_tokens": 10, + } + requestBody, err := json.Marshal(testRequest) + if err != nil { + logging.Error(caller+". Auto-test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + return + } + + completionsUrl := node.InferenceUrlWithVersion(version) + "/v1/chat/completions" + resp, err := http.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) + if err != nil { + logging.Error(caller+". Auto-test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + return + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + reason := fmt.Sprintf("non_success_status_code: %d", resp.StatusCode) + logging.Error(caller+". Auto-test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode, "error", reason) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) + return + } + } + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) logging.Info(caller+". Auto-test passed", types.Nodes, "node_id", node.Id) } diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index f3ecdcfe3..06c23d326 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -1,10 +1,13 @@ package admin import ( + "bytes" "context" "decentralized-api/apiconfig" "decentralized-api/logging" "decentralized-api/mlnodeclient" + "encoding/json" + "net/http" "time" "github.com/productscience/inference/x/inference/types" @@ -31,6 +34,13 @@ type TestResult struct { Metrics TestMetrics } +func getFirstModelId(models map[string]apiconfig.ModelConfig) string { + for modelId := range models { + return modelId + } + return "" // Return empty if no models +} + type MLnodeTestingOrchestrator struct { configManager *apiconfig.ConfigManager blockTimeSeconds float64 @@ -74,7 +84,33 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "health_not_ok", Metrics: metrics} } - metrics.RespMs = 0 + // Perform test inference request to validate response and measure performance + startResp := time.Now() + testRequest := map[string]interface{}{ + "model": getFirstModelId(node.Models), + "messages": []map[string]string{{"role": "user", "content": "Hello, how are you?"}}, + "max_tokens": 10, + } + requestBody, err := json.Marshal(testRequest) + if err != nil { + logging.Error("MLnode test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) + return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} + } + + completionsUrl := inferenceUrl + "/v1/chat/completions" + resp, err := http.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) + if err != nil { + logging.Error("MLnode test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) + return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + logging.Error("MLnode test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode) + return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "non_success_status_code", Metrics: metrics} + } + + metrics.RespMs = time.Since(startResp).Milliseconds() logging.Info("MLnode test succeeded", types.Nodes, "node_id", node.Id) return &TestResult{NodeId: node.Id, Status: TestSuccess, Metrics: metrics} From 5f74151416c607cea0d123a7dd6b58717b90b5c0 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 25 Dec 2025 00:09:18 +0800 Subject: [PATCH 19/31] add SetNodeMLNodeOnboardingStateCommand to handle mlnode state --- decentralized-api/broker/broker.go | 2 + .../broker/node_admin_commands.go | 61 +++++++++++++++++-- .../admin/mlnode_testing_orchestrator.go | 31 +++++++++- .../internal/server/admin/server.go | 2 +- 4 files changed, 88 insertions(+), 8 deletions(-) diff --git a/decentralized-api/broker/broker.go b/decentralized-api/broker/broker.go index 896dd20c6..81a7a28d7 100644 --- a/decentralized-api/broker/broker.go +++ b/decentralized-api/broker/broker.go @@ -502,6 +502,8 @@ func (b *Broker) executeCommand(command Command) { command.Execute(b) case SetNodeFailureReasonCommand: command.Execute(b) + case SetNodeMLNodeOnboardingStateCommand: + command.Execute(b) default: logging.Error("Unregistered command type", types.Nodes, "type", reflect.TypeOf(command).String()) } diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index e36b24569..5d5c06b8a 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "decentralized-api/apiconfig" + "decentralized-api/internal/server/admin" "decentralized-api/logging" "encoding/json" "fmt" @@ -388,11 +389,58 @@ func getFirstModelIdFromNode(node Node) string { return "" // Return empty if no models } +// SetNodeMLNodeOnboardingStateCommand updates the MLNodeOnboardingState of a node +type SetNodeMLNodeOnboardingStateCommand struct { + NodeId string + NewState string + Response chan bool +} + +func NewSetNodeMLNodeOnboardingStateCommand(nodeId string, newState string) SetNodeMLNodeOnboardingStateCommand { + return SetNodeMLNodeOnboardingStateCommand{ + NodeId: nodeId, + NewState: newState, + Response: make(chan bool, 2), + } +} + +func (c SetNodeMLNodeOnboardingStateCommand) GetResponseChannelCapacity() int { + return cap(c.Response) +} + +func (c SetNodeMLNodeOnboardingStateCommand) Execute(b *Broker) { + b.mu.Lock() + defer b.mu.Unlock() + + node, exists := b.nodes[c.NodeId] + if !exists { + logging.Error("Cannot set MLNodeOnboardingState: node not found", types.Nodes, "node_id", c.NodeId) + c.Response <- false + return + } + + logging.Info("Setting MLNodeOnboardingState for node", types.Nodes, + "node_id", c.NodeId, + "old_state", node.State.MLNodeOnboardingState, + "new_state", c.NewState) + + node.State.MLNodeOnboardingState = c.NewState + + c.Response <- true +} + // autoTestNodeIfTimeAllows performs a basic validation test for a node when // there is more than 1 hour remaining until the next PoC. It loads each // configured model and checks inference health, updating the node failure // reason accordingly. The caller string is used for logging context. func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { + // Helper function to handle test failures + setTestFailed := func(nodeId, errorReason string) { + cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(admin.MLNodeState_TEST_FAILED)) + _ = b.QueueMessage(cmd) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(nodeId, errorReason)) + } + epochState := b.phaseTracker.GetCurrentEpochState() if epochState == nil || !epochState.IsSynced { logging.Info(caller+". Auto-test skipped: epoch state unavailable", types.Nodes, "node_id", node.Id) @@ -419,7 +467,7 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { for modelId, cfg := range node.Models { if err := client.InferenceUp(ctx, modelId, cfg.Args); err != nil { logging.Error(caller+". Auto-test model load failed", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + setTestFailed(node.Id, err.Error()) return } } @@ -431,7 +479,7 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { reason = err.Error() } logging.Error(caller+". Auto-test health check failed", types.Nodes, "node_id", node.Id, "error", reason) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) + setTestFailed(node.Id, reason) return } @@ -446,7 +494,7 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { requestBody, err := json.Marshal(testRequest) if err != nil { logging.Error(caller+". Auto-test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + setTestFailed(node.Id, err.Error()) return } @@ -454,7 +502,7 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { resp, err := http.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) if err != nil { logging.Error(caller+". Auto-test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, err.Error())) + setTestFailed(node.Id, err.Error()) return } defer resp.Body.Close() @@ -462,11 +510,14 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { if resp.StatusCode < 200 || resp.StatusCode >= 300 { reason := fmt.Sprintf("non_success_status_code: %d", resp.StatusCode) logging.Error(caller+". Auto-test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode, "error", reason) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, reason)) + setTestFailed(node.Id, reason) return } } + // On success, set node MLNodeOnboardingState to WAITING_FOR_POC + cmd := NewSetNodeMLNodeOnboardingStateCommand(node.Id, string(admin.MLNodeState_WAITING_FOR_POC)) + _ = b.QueueMessage(cmd) _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) logging.Info(caller+". Auto-test passed", types.Nodes, "node_id", node.Id) } diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index 06c23d326..6a406cc12 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "decentralized-api/apiconfig" + "decentralized-api/broker" "decentralized-api/logging" "decentralized-api/mlnodeclient" "encoding/json" @@ -44,10 +45,11 @@ func getFirstModelId(models map[string]apiconfig.ModelConfig) string { type MLnodeTestingOrchestrator struct { configManager *apiconfig.ConfigManager blockTimeSeconds float64 + nodeBroker *broker.Broker } -func NewMLnodeTestingOrchestrator(cm *apiconfig.ConfigManager) *MLnodeTestingOrchestrator { - return &MLnodeTestingOrchestrator{configManager: cm, blockTimeSeconds: 6.0} +func NewMLnodeTestingOrchestrator(cm *apiconfig.ConfigManager, nodeBroker *broker.Broker) *MLnodeTestingOrchestrator { + return &MLnodeTestingOrchestrator{configManager: cm, blockTimeSeconds: 6.0, nodeBroker: nodeBroker} } func (o *MLnodeTestingOrchestrator) ShouldAutoTest(secondsUntilNextPoC int64) bool { @@ -55,6 +57,14 @@ func (o *MLnodeTestingOrchestrator) ShouldAutoTest(secondsUntilNextPoC int64) bo } func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apiconfig.InferenceNodeConfig) *TestResult { + // Helper function to set node state to TEST_FAILED on failure + setTestFailed := func(nodeId string) { + if o.nodeBroker != nil { + cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(MLNodeState_TEST_FAILED)) + _ = o.nodeBroker.QueueMessage(cmd) + } + } + version := o.configManager.GetCurrentNodeVersion() pocUrl := getPoCUrlWithVersion(node, version) inferenceUrl := formatURL(node.Host, node.InferencePort, node.InferenceSegment) @@ -68,6 +78,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon metrics.LoadMs[modelId] = time.Since(start).Milliseconds() if err != nil { logging.Error("MLnode test failed during model loading", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, FailingModel: modelId, Error: err.Error(), Metrics: metrics} } } @@ -78,9 +89,11 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon if err != nil || !ok { if err != nil { logging.Error("MLnode health check failed", types.Nodes, "node_id", node.Id, "error", err) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } logging.Error("MLnode health check not OK", types.Nodes, "node_id", node.Id) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "health_not_ok", Metrics: metrics} } @@ -94,6 +107,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon requestBody, err := json.Marshal(testRequest) if err != nil { logging.Error("MLnode test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } @@ -101,17 +115,30 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon resp, err := http.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) if err != nil { logging.Error("MLnode test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { logging.Error("MLnode test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode) + setTestFailed(node.Id) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "non_success_status_code", Metrics: metrics} } metrics.RespMs = time.Since(startResp).Milliseconds() + // Helper function to set node state to WAITING_FOR_POC on success + setTestSuccess := func(nodeId string) { + if o.nodeBroker != nil { + cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(MLNodeState_WAITING_FOR_POC)) + _ = o.nodeBroker.QueueMessage(cmd) + } + } + + // On success, set node MLNodeOnboardingState to WAITING_FOR_POC + setTestSuccess(node.Id) + logging.Info("MLnode test succeeded", types.Nodes, "node_id", node.Id) return &TestResult{NodeId: node.Id, Status: TestSuccess, Metrics: metrics} } diff --git a/decentralized-api/internal/server/admin/server.go b/decentralized-api/internal/server/admin/server.go index 50e53e7f0..d312d8ab1 100644 --- a/decentralized-api/internal/server/admin/server.go +++ b/decentralized-api/internal/server/admin/server.go @@ -62,7 +62,7 @@ func NewServer( payloadStorage: payloadStorage, onboarding: NewOnboardingStateManager(), statusReporter: NewStatusReporter(), - tester: NewMLnodeTestingOrchestrator(configManager), + tester: NewMLnodeTestingOrchestrator(configManager, nodeBroker), latestTestResults: map[string]*TestResult{}, } From 71bc21930dfdb32f3f5b906cf86c96ac3b2ee80d Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 25 Dec 2025 00:52:54 +0800 Subject: [PATCH 20/31] reslove cycle dependency for broker,main,admin --- decentralized-api/apiconfig/constants.go | 20 +++++++++++++++++++ .../broker/node_admin_commands.go | 5 ++--- .../admin/mlnode_testing_orchestrator.go | 4 ++-- .../server/admin/onboarding_state_manager.go | 19 +++++++++++------- 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 decentralized-api/apiconfig/constants.go diff --git a/decentralized-api/apiconfig/constants.go b/decentralized-api/apiconfig/constants.go new file mode 100644 index 000000000..3e4d0a267 --- /dev/null +++ b/decentralized-api/apiconfig/constants.go @@ -0,0 +1,20 @@ +package apiconfig + +// MLNodeOnboardingState represents the onboarding state of an ML node +type MLNodeOnboardingState string + +// Constants for MLNodeOnboardingState +const ( + MLNodeState_WAITING_FOR_POC MLNodeOnboardingState = "WAITING_FOR_POC" + MLNodeState_TESTING MLNodeOnboardingState = "TESTING" + MLNodeState_TEST_FAILED MLNodeOnboardingState = "TEST_FAILED" +) + +// ParticipantState represents the state of a participant +type ParticipantState string + +// Constants for ParticipantState +const ( + ParticipantState_INACTIVE_WAITING ParticipantState = "INACTIVE_WAITING" + ParticipantState_ACTIVE_PARTICIPATING ParticipantState = "ACTIVE_PARTICIPATING" +) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index 5d5c06b8a..89f123bf0 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "decentralized-api/apiconfig" - "decentralized-api/internal/server/admin" "decentralized-api/logging" "encoding/json" "fmt" @@ -436,7 +435,7 @@ func (c SetNodeMLNodeOnboardingStateCommand) Execute(b *Broker) { func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { // Helper function to handle test failures setTestFailed := func(nodeId, errorReason string) { - cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(admin.MLNodeState_TEST_FAILED)) + cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) _ = b.QueueMessage(cmd) _ = b.QueueMessage(NewSetNodeFailureReasonCommand(nodeId, errorReason)) } @@ -516,7 +515,7 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { } // On success, set node MLNodeOnboardingState to WAITING_FOR_POC - cmd := NewSetNodeMLNodeOnboardingStateCommand(node.Id, string(admin.MLNodeState_WAITING_FOR_POC)) + cmd := NewSetNodeMLNodeOnboardingStateCommand(node.Id, string(apiconfig.MLNodeState_WAITING_FOR_POC)) _ = b.QueueMessage(cmd) _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) logging.Info(caller+". Auto-test passed", types.Nodes, "node_id", node.Id) diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index 6a406cc12..0cc394690 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -60,7 +60,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon // Helper function to set node state to TEST_FAILED on failure setTestFailed := func(nodeId string) { if o.nodeBroker != nil { - cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(MLNodeState_TEST_FAILED)) + cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) _ = o.nodeBroker.QueueMessage(cmd) } } @@ -131,7 +131,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon // Helper function to set node state to WAITING_FOR_POC on success setTestSuccess := func(nodeId string) { if o.nodeBroker != nil { - cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(MLNodeState_WAITING_FOR_POC)) + cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_WAITING_FOR_POC)) _ = o.nodeBroker.QueueMessage(cmd) } } diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index 069eb0808..5972d9bcc 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -1,18 +1,23 @@ package admin -type MLNodeOnboardingState string +import "decentralized-api/apiconfig" + +// Import types and constants from apiconfig package +// These are defined there to avoid import cycles + +type MLNodeOnboardingState = apiconfig.MLNodeOnboardingState const ( - MLNodeState_WAITING_FOR_POC MLNodeOnboardingState = "WAITING_FOR_POC" - MLNodeState_TESTING MLNodeOnboardingState = "TESTING" - MLNodeState_TEST_FAILED MLNodeOnboardingState = "TEST_FAILED" + MLNodeState_WAITING_FOR_POC = apiconfig.MLNodeState_WAITING_FOR_POC + MLNodeState_TESTING = apiconfig.MLNodeState_TESTING + MLNodeState_TEST_FAILED = apiconfig.MLNodeState_TEST_FAILED ) -type ParticipantState string +type ParticipantState = apiconfig.ParticipantState const ( - ParticipantState_INACTIVE_WAITING ParticipantState = "INACTIVE_WAITING" - ParticipantState_ACTIVE_PARTICIPATING ParticipantState = "ACTIVE_PARTICIPATING" + ParticipantState_INACTIVE_WAITING = apiconfig.ParticipantState_INACTIVE_WAITING + ParticipantState_ACTIVE_PARTICIPATING = apiconfig.ParticipantState_ACTIVE_PARTICIPATING ) type OnboardingStateManager struct { From 89b0402d9dbb195e69afa90d8c94d06025afed8a Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 25 Dec 2025 12:36:31 +0800 Subject: [PATCH 21/31] enhance log info --- decentralized-api/broker/commands.go | 10 ++++++++++ .../internal/event_listener/new_block_dispatcher.go | 9 +++++++++ .../internal/server/admin/node_handlers.go | 8 ++++++++ 3 files changed, 27 insertions(+) diff --git a/decentralized-api/broker/commands.go b/decentralized-api/broker/commands.go index 1e21cca7b..30ab176c2 100644 --- a/decentralized-api/broker/commands.go +++ b/decentralized-api/broker/commands.go @@ -272,6 +272,9 @@ func (c UpdateNodeResultCommand) Execute(b *Broker) { return } + prevStatus := node.State.CurrentStatus + prevFailure := node.State.FailureReason + // Update state logging.Info("Finalizing state transition for node", types.Nodes, "node_id", c.NodeId, @@ -290,6 +293,9 @@ func (c UpdateNodeResultCommand) Execute(b *Broker) { } else { // Clear failure reason on success node.State.FailureReason = "" + if prevFailure != "" { + logging.Info("Node status recovered", types.Nodes, "node_id", c.NodeId, "blockHeight", blockHeight) + } } // Reset POC fields when moving away from POC status @@ -298,6 +304,10 @@ func (c UpdateNodeResultCommand) Execute(b *Broker) { node.State.PocCurrentStatus = PocStatusIdle } + if prevStatus == types.HardwareNodeStatus_POC && c.Result.FinalStatus == types.HardwareNodeStatus_INFERENCE { + logging.Info("Onboarding transition POC->INFERENCE", types.Nodes, "node_id", c.NodeId, "blockHeight", blockHeight) + } + c.Response <- true } diff --git a/decentralized-api/internal/event_listener/new_block_dispatcher.go b/decentralized-api/internal/event_listener/new_block_dispatcher.go index 8bf1b0446..0786200cc 100644 --- a/decentralized-api/internal/event_listener/new_block_dispatcher.go +++ b/decentralized-api/internal/event_listener/new_block_dispatcher.go @@ -16,7 +16,9 @@ import ( "decentralized-api/cosmosclient" "decentralized-api/internal" "decentralized-api/internal/event_listener/chainevents" + "decentralized-api/internal/poc" "decentralized-api/internal/seed" + "decentralized-api/internal/server/admin" "decentralized-api/internal/validation" "decentralized-api/logging" "decentralized-api/poc" @@ -336,6 +338,9 @@ func (d *OnNewBlockDispatcher) handlePhaseTransitions(epochState chainphase.Epoc epochContext := epochState.LatestEpoch blockHeight := epochState.CurrentBlock.Height blockHash := epochState.CurrentBlock.Hash + tc := admin.NewTimingCalculator() + tr := tc.TimeUntilNextPoC(&epochState, 6.0) + sr := admin.NewStatusReporter() // Sync broker node state with the latest epoch data at the start of a transition check if err := d.nodeBroker.UpdateNodeWithEpochData(&epochState); err != nil { @@ -346,6 +351,7 @@ func (d *OnNewBlockDispatcher) handlePhaseTransitions(epochState chainphase.Epoc // Check for PoC start for the next epoch. This is the most important transition. if epochContext.IsStartOfPocStage(blockHeight) { logging.Info("DapiStage:IsStartOfPocStage: sending StartPoCEvent to the PoC orchestrator", types.Stages, "blockHeight", blockHeight, "blockHash", blockHash) + sr.LogTimingGuidance(tr.SecondsUntilNextPoC) d.randomSeedManager.GenerateSeedInfo(epochContext.EpochIndex) return } @@ -354,6 +360,7 @@ func (d *OnNewBlockDispatcher) handlePhaseTransitions(epochState chainphase.Epoc if epochContext.IsEndOfPoCStage(blockHeight) { logging.Info("DapiStage:IsEndOfPoCStage. Calling MoveToValidationStage", types.Stages, "blockHeigh", blockHeight, "blockHash", blockHash) + sr.LogTimingGuidance(tr.SecondsUntilNextPoC) command := broker.NewInitValidateCommand() err := d.nodeBroker.QueueMessage(command) if err != nil { @@ -365,6 +372,7 @@ func (d *OnNewBlockDispatcher) handlePhaseTransitions(epochState chainphase.Epoc if epochContext.IsStartOfPoCValidationStage(blockHeight) { logging.Info("DapiStage:IsStartOfPoCValidationStage", types.Stages, "blockHeight", blockHeight, "blockHash", blockHash, "pocStartBlockHeight", epochContext.PocStartBlockHeight) pocStartBlockHeight := epochContext.PocStartBlockHeight + sr.LogTimingGuidance(tr.SecondsUntilNextPoC) go func() { pocStartBlockHash, err := d.nodeBroker.GetChainBridge().GetBlockHash(pocStartBlockHeight) if err != nil { @@ -378,6 +386,7 @@ func (d *OnNewBlockDispatcher) handlePhaseTransitions(epochState chainphase.Epoc if epochContext.IsEndOfPoCValidationStage(blockHeight) { logging.Info("DapiStage:IsEndOfPoCValidationStage", types.Stages, "blockHeight", blockHeight, "blockHash", blockHash) + sr.LogTimingGuidance(tr.SecondsUntilNextPoC) command := broker.NewInferenceUpAllCommand() err := d.nodeBroker.QueueMessage(command) if err != nil { diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 6df1d6363..34d6706df 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -32,6 +32,10 @@ func (s *Server) getNodes(ctx echo.Context) error { state := &nodes[i].State participantActive := chainActive || len(state.EpochMLNodes) > 0 pstate := osm.ParticipantStatus(participantActive) + prevParticipant := ParticipantState(state.ParticipantState) + if prevParticipant != pstate { + sr.LogParticipantStatusChange(prevParticipant, pstate) + } state.ParticipantState = string(pstate) var secs int64 @@ -52,6 +56,10 @@ func (s *Server) getNodes(ctx echo.Context) error { userMsg = sr.BuildMLNodeMessage(mlnodeState, secs, "") } + prevOnboarding := MLNodeOnboardingState(state.MLNodeOnboardingState) + if prevOnboarding != mlnodeState { + sr.LogOnboardingTransition(prevOnboarding, mlnodeState) + } state.MLNodeOnboardingState = string(mlnodeState) state.UserMessage = userMsg state.Guidance = sr.BuildParticipantMessage(pstate) From b5243e7f1046b7613d362d8705f53116f9a2c64f Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Thu, 25 Dec 2025 13:09:37 +0800 Subject: [PATCH 22/31] TEST_FAILED state with detailed error reporting in MLnode --- .../broker/node_admin_commands.go | 20 +- .../admin/mlnode_testing_orchestrator.go | 30 +- .../internal/server/admin/server_test.go | 18 + decentralized-api/mlnodeclient/client.go | 27 ++ decentralized-api/mlnodeclient/interface.go | 1 + decentralized-api/mlnodeclient/mock.go | 355 +++++------------- 6 files changed, 173 insertions(+), 278 deletions(-) diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index 89f123bf0..3101d1273 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -5,6 +5,7 @@ import ( "context" "decentralized-api/apiconfig" "decentralized-api/logging" + "decentralized-api/mlnodeclient" "encoding/json" "fmt" "net/http" @@ -433,13 +434,6 @@ func (c SetNodeMLNodeOnboardingStateCommand) Execute(b *Broker) { // configured model and checks inference health, updating the node failure // reason accordingly. The caller string is used for logging context. func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { - // Helper function to handle test failures - setTestFailed := func(nodeId, errorReason string) { - cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) - _ = b.QueueMessage(cmd) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(nodeId, errorReason)) - } - epochState := b.phaseTracker.GetCurrentEpochState() if epochState == nil || !epochState.IsSynced { logging.Info(caller+". Auto-test skipped: epoch state unavailable", types.Nodes, "node_id", node.Id) @@ -460,6 +454,18 @@ func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) logging.Info(caller+". Auto-test starting", types.Nodes, "node_id", node.Id, "models", len(node.Models)) + // Helper function to handle test failures + setTestFailed := func(nodeId, errorReason string) { + cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) + _ = b.QueueMessage(cmd) + _ = b.QueueMessage(NewSetNodeFailureReasonCommand(nodeId, errorReason)) + + // Notify MLnode about the failure + notifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.SetNodeState(notifyCtx, mlnodeclient.MlNodeState_TEST_FAILED, errorReason) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index 0cc394690..d8f7888f4 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -57,18 +57,24 @@ func (o *MLnodeTestingOrchestrator) ShouldAutoTest(secondsUntilNextPoC int64) bo } func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apiconfig.InferenceNodeConfig) *TestResult { + version := o.configManager.GetCurrentNodeVersion() + pocUrl := getPoCUrlWithVersion(node, version) + inferenceUrl := formatURL(node.Host, node.InferencePort, node.InferenceSegment) + client := mlnodeclient.NewNodeClient(pocUrl, inferenceUrl) + // Helper function to set node state to TEST_FAILED on failure - setTestFailed := func(nodeId string) { + setTestFailed := func(nodeId string, reason string) { if o.nodeBroker != nil { cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) _ = o.nodeBroker.QueueMessage(cmd) + _ = o.nodeBroker.QueueMessage(broker.NewSetNodeFailureReasonCommand(nodeId, reason)) } - } - version := o.configManager.GetCurrentNodeVersion() - pocUrl := getPoCUrlWithVersion(node, version) - inferenceUrl := formatURL(node.Host, node.InferencePort, node.InferenceSegment) - client := mlnodeclient.NewNodeClient(pocUrl, inferenceUrl) + // Notify MLnode about the failure + notifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.SetNodeState(notifyCtx, mlnodeclient.MlNodeState_TEST_FAILED, reason) + } metrics := TestMetrics{LoadMs: map[string]int64{}} @@ -78,7 +84,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon metrics.LoadMs[modelId] = time.Since(start).Milliseconds() if err != nil { logging.Error("MLnode test failed during model loading", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) - setTestFailed(node.Id) + setTestFailed(node.Id, err.Error()) return &TestResult{NodeId: node.Id, Status: TestFailed, FailingModel: modelId, Error: err.Error(), Metrics: metrics} } } @@ -89,11 +95,11 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon if err != nil || !ok { if err != nil { logging.Error("MLnode health check failed", types.Nodes, "node_id", node.Id, "error", err) - setTestFailed(node.Id) + setTestFailed(node.Id, err.Error()) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } logging.Error("MLnode health check not OK", types.Nodes, "node_id", node.Id) - setTestFailed(node.Id) + setTestFailed(node.Id, "health_not_ok") return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "health_not_ok", Metrics: metrics} } @@ -107,7 +113,7 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon requestBody, err := json.Marshal(testRequest) if err != nil { logging.Error("MLnode test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) - setTestFailed(node.Id) + setTestFailed(node.Id, err.Error()) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } @@ -115,14 +121,14 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon resp, err := http.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) if err != nil { logging.Error("MLnode test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) - setTestFailed(node.Id) + setTestFailed(node.Id, err.Error()) return &TestResult{NodeId: node.Id, Status: TestFailed, Error: err.Error(), Metrics: metrics} } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { logging.Error("MLnode test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode) - setTestFailed(node.Id) + setTestFailed(node.Id, "non_success_status_code") return &TestResult{NodeId: node.Id, Status: TestFailed, Error: "non_success_status_code", Metrics: metrics} } diff --git a/decentralized-api/internal/server/admin/server_test.go b/decentralized-api/internal/server/admin/server_test.go index 9e40173e9..3880979c8 100644 --- a/decentralized-api/internal/server/admin/server_test.go +++ b/decentralized-api/internal/server/admin/server_test.go @@ -60,6 +60,14 @@ func (m *mockInferenceQueryClient) EpochGroupData(ctx context.Context, in *types return args.Get(0).(*types.QueryGetEpochGroupDataResponse), args.Error(1) } +func (m *mockInferenceQueryClient) CurrentEpochGroupData(ctx context.Context, in *types.QueryCurrentEpochGroupDataRequest, opts ...grpc.CallOption) (*types.QueryCurrentEpochGroupDataResponse, error) { + args := m.Called(ctx, in) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*types.QueryCurrentEpochGroupDataResponse), args.Error(1) +} + func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodeclient.MockClientFactory, *chainphase.ChainPhaseTracker) { // Disable model enforcement in tests os.Setenv("ENFORCED_MODEL_ID", "disabled") @@ -95,6 +103,16 @@ func setupTestServer(t *testing.T) (*Server, *apiconfig.ConfigManager, *mlnodecl mockParticipant.On("GetAddress").Return("test-participant") mockCosmos.On("GetContext").Return(context.Background()) + // Mock CurrentEpochGroupData + currentEpochResp := &types.QueryCurrentEpochGroupDataResponse{ + EpochGroupData: types.EpochGroupData{ + PocStartBlockHeight: 100, + EpochIndex: 100, + SubGroupModels: []string{"test-model"}, + }, + } + mockQueryClient.On("CurrentEpochGroupData", mock.Anything, mock.Anything).Return(currentEpochResp, nil) + // Mock epoch group data for parent group (empty modelId) parentGroupResp := &types.QueryGetEpochGroupDataResponse{ EpochGroupData: types.EpochGroupData{ diff --git a/decentralized-api/mlnodeclient/client.go b/decentralized-api/mlnodeclient/client.go index 3aaa90b1a..0893f37a3 100644 --- a/decentralized-api/mlnodeclient/client.go +++ b/decentralized-api/mlnodeclient/client.go @@ -210,8 +210,35 @@ const ( MlNodeState_INFERENCE MLNodeState = "INFERENCE" MlNodeState_TRAIN MLNodeState = "TRAIN" MlNodeState_STOPPED MLNodeState = "STOPPED" + MlNodeState_TEST_FAILED MLNodeState = "TEST_FAILED" ) +type SetStateRequest struct { + State MLNodeState `json:"state"` + Error string `json:"error,omitempty"` +} + +func (api *Client) SetNodeState(ctx context.Context, state MLNodeState, errorReason string) error { + requestURL, err := url.JoinPath(api.pocUrl, nodeStatePath) + if err != nil { + return err + } + + body := SetStateRequest{ + State: state, + Error: errorReason, + } + + logging.Info("Sending state update to MLnode", types.Nodes, "url", requestURL, "state", state, "error", errorReason) + + _, err = utils.SendPostJsonRequest(ctx, &api.client, requestURL, body) + if err != nil { + logging.Error("Failed to set MLnode state", types.Nodes, "error", err) + return err + } + return nil +} + type StateResponse struct { State MLNodeState `json:"state"` } diff --git a/decentralized-api/mlnodeclient/interface.go b/decentralized-api/mlnodeclient/interface.go index 64f55cc8b..f53400d55 100644 --- a/decentralized-api/mlnodeclient/interface.go +++ b/decentralized-api/mlnodeclient/interface.go @@ -11,6 +11,7 @@ type MLNodeClient interface { // Node state operations Stop(ctx context.Context) error NodeState(ctx context.Context) (*StateResponse, error) + SetNodeState(ctx context.Context, state MLNodeState, errorReason string) error // PoC v1 operations (on-chain batches, requires Stop before transitions) InitGenerateV1(ctx context.Context, dto InitDtoV1) error diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index de9379b72..5d310dc4b 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -3,6 +3,7 @@ package mlnodeclient import ( "context" "decentralized-api/logging" + "errors" "sync" "testing" @@ -29,6 +30,11 @@ type MockClient struct { // Error injection StopError error NodeStateError error + SetNodeStateError error + GetPowStatusError error + InitGenerateError error + InitValidateError error + ValiateBatchError error InferenceHealthError error InferenceUpError error StartTrainingError error @@ -39,14 +45,15 @@ type MockClient struct { DeleteModelError error ListModelsError error GetDiskSpaceError error - InitGenerateV1Error error - InitValidateV1Error error - ValidateBatchV1Error error - GetPowStatusV1Error error // Call tracking StopCalled int NodeStateCalled int + SetNodeStateCalled int + GetPowStatusCalled int + InitGenerateCalled int + InitValidateCalled int + ValidateBatchCalled int InferenceHealthCalled int InferenceUpCalled int StartTrainingCalled int @@ -58,31 +65,13 @@ type MockClient struct { ListModelsCalled int GetDiskSpaceCalled int - // PoC v1 call tracking - InitGenerateV1Called int - InitValidateV1Called int - ValidateBatchV1Called int - GetPowStatusV1Called int - - // PoC v2 call tracking - InitGenerateV2Called int - GenerateV2Called int - GetPowStatusV2Called int - StopPowV2Called int - - // Track Init/Validate attempts (for testing) - InitValidateCalled int - - // PoC v1 state - PowStatusV1 PowStateV1 // V1 status enum - - // PoC v2 state - PowStatusV2 string // "IDLE", "GENERATING", etc. - // Capture parameters - LastInferenceModel string - LastInferenceArgs []string - LastTrainingParams struct { + LastInitDto *InitDto + LastInitValidateDto *InitDto + LastValidateBatch ProofBatch + LastInferenceModel string + LastInferenceArgs []string + LastTrainingParams struct { TaskId uint64 Participant string NodeId string @@ -93,6 +82,8 @@ type MockClient struct { LastModelStatusCheck *Model LastModelDownload *Model LastModelDelete *Model + LastSetNodeState MLNodeState + LastSetNodeReason string } // NewMockClient creates a new mock client with default values @@ -118,121 +109,113 @@ func (m *MockClient) WithTryLock(t *testing.T, f func()) { f() } -func (m *MockClient) GetInferenceUpCalled() int { +func (m *MockClient) Stop(ctx context.Context) error { m.Mu.Lock() defer m.Mu.Unlock() - return m.InferenceUpCalled + logging.Info("MockClient. Stop: called", types.Testing) + m.StopCalled++ + if m.StopError != nil { + return m.StopError + } + m.CurrentState = MlNodeState_STOPPED + m.PowStatus = POW_STOPPED + m.InferenceIsHealthy = false + return nil } -func (m *MockClient) GetStopCalled() int { +func (m *MockClient) NodeState(ctx context.Context) (*StateResponse, error) { m.Mu.Lock() defer m.Mu.Unlock() - return m.StopCalled + m.NodeStateCalled++ + if m.NodeStateError != nil { + return nil, m.NodeStateError + } + return &StateResponse{State: m.CurrentState}, nil } -func (m *MockClient) GetNodeStateCalled() int { +func (m *MockClient) SetNodeState(ctx context.Context, state MLNodeState, errorReason string) error { m.Mu.Lock() defer m.Mu.Unlock() - return m.NodeStateCalled + m.SetNodeStateCalled++ + m.LastSetNodeState = state + m.LastSetNodeReason = errorReason + + if m.SetNodeStateError != nil { + return m.SetNodeStateError + } + m.CurrentState = state + return nil } -func (m *MockClient) GetInferenceHealthCalled() int { +func (m *MockClient) GetPowStatus(ctx context.Context) (*PowStatusResponse, error) { m.Mu.Lock() defer m.Mu.Unlock() - return m.InferenceHealthCalled + m.GetPowStatusCalled++ + if m.GetPowStatusError != nil { + return nil, m.GetPowStatusError + } + return &PowStatusResponse{ + Status: m.PowStatus, + IsModelInitialized: m.PowStatus == POW_GENERATING, + }, nil } -func (m *MockClient) Reset() { +func (m *MockClient) InitGenerate(ctx context.Context, dto InitDto) error { m.Mu.Lock() defer m.Mu.Unlock() - m.CurrentState = MlNodeState_STOPPED - m.PowStatus = POW_STOPPED - m.InferenceIsHealthy = false - m.GPUDevices = []GPUDevice{} - m.DriverInfo = nil - m.CachedModels = make(map[string]ModelListItem) - m.DownloadingModels = make(map[string]*DownloadProgress) - m.DiskSpace = nil - - m.StopError = nil - m.NodeStateError = nil - m.InferenceHealthError = nil - m.InferenceUpError = nil - m.StartTrainingError = nil - m.GetGPUDevicesError = nil - m.GetGPUDriverError = nil - m.CheckModelStatusError = nil - m.DownloadModelError = nil - m.DeleteModelError = nil - m.ListModelsError = nil - m.GetDiskSpaceError = nil - m.InitGenerateV1Error = nil - m.InitValidateV1Error = nil - m.ValidateBatchV1Error = nil - m.GetPowStatusV1Error = nil - - m.StopCalled = 0 - m.NodeStateCalled = 0 - m.InitValidateCalled = 0 - m.InferenceHealthCalled = 0 - m.InferenceUpCalled = 0 - m.StartTrainingCalled = 0 - m.GetGPUDevicesCalled = 0 - m.GetGPUDriverCalled = 0 - m.CheckModelStatusCalled = 0 - m.DownloadModelCalled = 0 - m.DeleteModelCalled = 0 - m.ListModelsCalled = 0 - m.GetDiskSpaceCalled = 0 - m.InitGenerateV1Called = 0 - m.InitValidateV1Called = 0 - m.ValidateBatchV1Called = 0 - m.GetPowStatusV1Called = 0 - m.InitGenerateV2Called = 0 - m.GenerateV2Called = 0 - m.GetPowStatusV2Called = 0 - m.StopPowV2Called = 0 - - m.LastInferenceModel = "" - m.LastInferenceArgs = nil - m.LastTrainingParams = struct { - TaskId uint64 - Participant string - NodeId string - MasterNodeAddr string - Rank int - WorldSize int - }{} - m.LastModelStatusCheck = nil - m.LastModelDownload = nil - m.LastModelDelete = nil - m.PowStatusV1 = "" - m.PowStatusV2 = "" + if m.CurrentState != MlNodeState_STOPPED { + return errors.New("InitGenerate called with invalid state. Expected STOPPED. Actual: currentState =" + string(m.CurrentState)) + } + + logging.Info("MockClient. InitGenerate: called", types.Testing) + m.InitGenerateCalled++ + m.LastInitDto = &dto + if m.InitGenerateError != nil { + return m.InitGenerateError + } + m.CurrentState = MlNodeState_POW + m.PowStatus = POW_GENERATING + return nil } -func (m *MockClient) Stop(ctx context.Context) error { +func (m *MockClient) InitValidate(ctx context.Context, dto InitDto) error { m.Mu.Lock() defer m.Mu.Unlock() - logging.Info("MockClient. Stop: called", types.Testing) - m.StopCalled++ - if m.StopError != nil { - return m.StopError + + if m.CurrentState != MlNodeState_POW || + m.PowStatus != POW_GENERATING { + return errors.New("InitValidate called with invalid state. Expected MlNodeState_POW and POW_GENERATING. Actual: currentState = " + string(m.CurrentState) + ". powStatus =" + string(m.PowStatus)) } - m.CurrentState = MlNodeState_STOPPED - m.PowStatus = POW_STOPPED - m.InferenceIsHealthy = false + + logging.Info("MockClient. InitValidate: called", types.Testing) + m.InitValidateCalled++ + m.LastInitValidateDto = &dto + if m.InitValidateError != nil { + return m.InitValidateError + } + m.CurrentState = MlNodeState_POW + m.PowStatus = POW_VALIDATING return nil } -func (m *MockClient) NodeState(ctx context.Context) (*StateResponse, error) { +func (m *MockClient) ValidateBatch(ctx context.Context, batch ProofBatch) error { m.Mu.Lock() defer m.Mu.Unlock() - m.NodeStateCalled++ - if m.NodeStateError != nil { - return nil, m.NodeStateError + + if m.CurrentState != MlNodeState_POW || + m.PowStatus != POW_VALIDATING { + return errors.New("ValidateBatch called with invalid state. Expected MlNodeState_POW and POW_VALIDATING. Actual: currentState = " + string(m.CurrentState) + ". powStatus =" + string(m.PowStatus)) } - return &StateResponse{State: m.CurrentState}, nil + + m.ValidateBatchCalled++ + m.LastValidateBatch = batch + if m.ValiateBatchError != nil { + return m.ValiateBatchError + } + m.CurrentState = MlNodeState_POW + m.PowStatus = POW_VALIDATING + return nil } func (m *MockClient) InferenceHealth(ctx context.Context) (bool, error) { @@ -259,16 +242,6 @@ func (m *MockClient) InferenceUp(ctx context.Context, model string, args []strin return nil } -func (m *MockClient) GetLoadedModels(ctx context.Context) ([]string, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - // Return the last inference model that was loaded, if any - if m.LastInferenceModel != "" { - return []string{m.LastInferenceModel}, nil - } - return nil, nil -} - func (m *MockClient) StartTraining(ctx context.Context, taskId uint64, participant string, nodeId string, masterNodeAddr string, rank int, worldSize int) error { m.Mu.Lock() defer m.Mu.Unlock() @@ -293,68 +266,6 @@ func (m *MockClient) GetTrainingStatus(ctx context.Context) error { return nil } -// PoC v1 mock methods - -func (m *MockClient) InitGenerateV1(ctx context.Context, dto InitDtoV1) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.InitGenerateV1Called++ - if m.InitGenerateV1Error != nil { - return m.InitGenerateV1Error - } - - m.CurrentState = MlNodeState_POW - m.PowStatusV1 = PowStateV1Generating - m.InferenceIsHealthy = false - return nil -} - -func (m *MockClient) InitValidateV1(ctx context.Context, dto InitDtoV1) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.InitValidateV1Called++ - m.InitValidateCalled++ - if m.InitValidateV1Error != nil { - return m.InitValidateV1Error - } - - m.CurrentState = MlNodeState_POW - m.PowStatusV1 = PowStateV1Validating - return nil -} - -func (m *MockClient) ValidateBatchV1(ctx context.Context, batch ProofBatchV1) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.ValidateBatchV1Called++ - if m.ValidateBatchV1Error != nil { - return m.ValidateBatchV1Error - } - return nil -} - -func (m *MockClient) GetPowStatusV1(ctx context.Context) (*PowStatusResponseV1, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.GetPowStatusV1Called++ - if m.GetPowStatusV1Error != nil { - return nil, m.GetPowStatusV1Error - } - - status := m.PowStatusV1 - if status == "" { - status = PowStateV1Idle - } - return &PowStatusResponseV1{ - Status: status, - IsModelInitialized: m.CurrentState == MlNodeState_POW, - }, nil -} - // GPU operations func (m *MockClient) GetGPUDevices(ctx context.Context) (*GPUDevicesResponse, error) { @@ -520,79 +431,5 @@ func getModelKey(model Model) string { return model.HfRepo + ":latest" } -// PoC v2 mock methods - -func (m *MockClient) InitGenerateV2(ctx context.Context, req PoCInitGenerateRequestV2) (*PoCInitGenerateResponseV2, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.InitGenerateV2Called++ - - // Update mock state: node is now in PoC generation mode, not inference - m.CurrentState = MlNodeState_POW - m.InferenceIsHealthy = false - - // Default success response - return &PoCInitGenerateResponseV2{ - Status: "OK", - Backends: 1, - NGroups: 1, - }, nil -} - -func (m *MockClient) GenerateV2(ctx context.Context, req PoCGenerateRequestV2) (*PoCGenerateResponseV2, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.GenerateV2Called++ - - // Default success response - return &PoCGenerateResponseV2{ - Status: "queued", - RequestId: "mock-request-id", - }, nil -} - -func (m *MockClient) GetPowStatusV2(ctx context.Context) (*PoCStatusResponseV2, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.GetPowStatusV2Called++ - - // Use configured status or default to IDLE - status := m.PowStatusV2 - if status == "" { - status = "IDLE" - } - return &PoCStatusResponseV2{ - Status: status, - Backends: []BackendStatusV2{ - {Port: 8000, Status: status}, - }, - }, nil -} - -func (m *MockClient) StopPowV2(ctx context.Context) (*PoCStopResponseV2, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - - m.StopPowV2Called++ - - // Default success response - return &PoCStopResponseV2{ - Status: "OK", - Results: []BackendResult{ - {Port: 8000, Status: "stopped"}, - }, - }, nil -} - -// SetV2Status sets the v2 status for testing -func (m *MockClient) SetV2Status(status string) { - m.Mu.Lock() - defer m.Mu.Unlock() - m.PowStatusV2 = status -} - // Ensure MockClient implements MLNodeClient var _ MLNodeClient = (*MockClient)(nil) From 576b3db136c6a6bdc5ac882d1a91b8135347fbb2 Mon Sep 17 00:00:00 2001 From: Johnny Feng Date: Wed, 4 Feb 2026 13:29:12 +0800 Subject: [PATCH 23/31] funcs been replaced with their v1 version --- decentralized-api/mlnodeclient/mock.go | 72 -------------------------- 1 file changed, 72 deletions(-) diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index 5d310dc4b..72d9ace96 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -3,7 +3,6 @@ package mlnodeclient import ( "context" "decentralized-api/logging" - "errors" "sync" "testing" @@ -147,77 +146,6 @@ func (m *MockClient) SetNodeState(ctx context.Context, state MLNodeState, errorR return nil } -func (m *MockClient) GetPowStatus(ctx context.Context) (*PowStatusResponse, error) { - m.Mu.Lock() - defer m.Mu.Unlock() - m.GetPowStatusCalled++ - if m.GetPowStatusError != nil { - return nil, m.GetPowStatusError - } - return &PowStatusResponse{ - Status: m.PowStatus, - IsModelInitialized: m.PowStatus == POW_GENERATING, - }, nil -} - -func (m *MockClient) InitGenerate(ctx context.Context, dto InitDto) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - if m.CurrentState != MlNodeState_STOPPED { - return errors.New("InitGenerate called with invalid state. Expected STOPPED. Actual: currentState =" + string(m.CurrentState)) - } - - logging.Info("MockClient. InitGenerate: called", types.Testing) - m.InitGenerateCalled++ - m.LastInitDto = &dto - if m.InitGenerateError != nil { - return m.InitGenerateError - } - m.CurrentState = MlNodeState_POW - m.PowStatus = POW_GENERATING - return nil -} - -func (m *MockClient) InitValidate(ctx context.Context, dto InitDto) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - if m.CurrentState != MlNodeState_POW || - m.PowStatus != POW_GENERATING { - return errors.New("InitValidate called with invalid state. Expected MlNodeState_POW and POW_GENERATING. Actual: currentState = " + string(m.CurrentState) + ". powStatus =" + string(m.PowStatus)) - } - - logging.Info("MockClient. InitValidate: called", types.Testing) - m.InitValidateCalled++ - m.LastInitValidateDto = &dto - if m.InitValidateError != nil { - return m.InitValidateError - } - m.CurrentState = MlNodeState_POW - m.PowStatus = POW_VALIDATING - return nil -} - -func (m *MockClient) ValidateBatch(ctx context.Context, batch ProofBatch) error { - m.Mu.Lock() - defer m.Mu.Unlock() - - if m.CurrentState != MlNodeState_POW || - m.PowStatus != POW_VALIDATING { - return errors.New("ValidateBatch called with invalid state. Expected MlNodeState_POW and POW_VALIDATING. Actual: currentState = " + string(m.CurrentState) + ". powStatus =" + string(m.PowStatus)) - } - - m.ValidateBatchCalled++ - m.LastValidateBatch = batch - if m.ValiateBatchError != nil { - return m.ValiateBatchError - } - m.CurrentState = MlNodeState_POW - m.PowStatus = POW_VALIDATING - return nil -} - func (m *MockClient) InferenceHealth(ctx context.Context) (bool, error) { m.Mu.Lock() defer m.Mu.Unlock() From 83b06825231d17dc7a8f4b559ddd3bb98b8395af Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Sat, 28 Feb 2026 14:04:29 +0800 Subject: [PATCH 24/31] fix:Make it Compile --- .../event_listener/new_block_dispatcher.go | 2 +- .../server/admin/onboarding_state_manager.go | 21 ++++---- decentralized-api/mlnodeclient/mock.go | 20 ++++++++ decentralized-api/poc/validator_v1_test.go | 8 +++ fix plan.md | 51 +++++++++++++++++++ 5 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 fix plan.md diff --git a/decentralized-api/internal/event_listener/new_block_dispatcher.go b/decentralized-api/internal/event_listener/new_block_dispatcher.go index 0786200cc..740a5c866 100644 --- a/decentralized-api/internal/event_listener/new_block_dispatcher.go +++ b/decentralized-api/internal/event_listener/new_block_dispatcher.go @@ -16,7 +16,7 @@ import ( "decentralized-api/cosmosclient" "decentralized-api/internal" "decentralized-api/internal/event_listener/chainevents" - "decentralized-api/internal/poc" + // "decentralized-api/internal/poc" // 移除无效导入 "decentralized-api/internal/seed" "decentralized-api/internal/server/admin" "decentralized-api/internal/validation" diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index 5972d9bcc..bd63795a4 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -97,13 +97,16 @@ func itoa(v int64) string { } func fmtInt(v int64) string { - var buf [20]byte - i := len(buf) - n := v - for n > 0 { - i-- - buf[i] = byte('0' + n%10) - n /= 10 - } - return string(buf[i:]) + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + n := v + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) } diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index 72d9ace96..32eb6ad62 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -85,6 +85,26 @@ type MockClient struct { LastSetNodeReason string } +// Stub for missing MLNodeClient methods +func (m *MockClient) InitGenerateV1(ctx context.Context, dto InitDtoV1) error { return nil } +func (m *MockClient) InitValidateV1(ctx context.Context, dto InitDtoV1) error { return nil } +func (m *MockClient) ValidateBatchV1(ctx context.Context, batch ProofBatchV1) error { return nil } +func (m *MockClient) GetPowStatusV1(ctx context.Context) (*PowStatusResponseV1, error) { + return &PowStatusResponseV1{}, nil +} +func (m *MockClient) InitGenerateV2(ctx context.Context, req PoCInitGenerateRequestV2) (*PoCInitGenerateResponseV2, error) { + return &PoCInitGenerateResponseV2{}, nil +} +func (m *MockClient) GenerateV2(ctx context.Context, req PoCGenerateRequestV2) (*PoCGenerateResponseV2, error) { + return &PoCGenerateResponseV2{}, nil +} +func (m *MockClient) GetPowStatusV2(ctx context.Context) (*PoCStatusResponseV2, error) { + return &PoCStatusResponseV2{}, nil +} +func (m *MockClient) StopPowV2(ctx context.Context) (*PoCStopResponseV2, error) { + return &PoCStopResponseV2{}, nil +} + // NewMockClient creates a new mock client with default values func NewMockClient() *MockClient { return &MockClient{ diff --git a/decentralized-api/poc/validator_v1_test.go b/decentralized-api/poc/validator_v1_test.go index f7af79de1..6a10e01e1 100644 --- a/decentralized-api/poc/validator_v1_test.go +++ b/decentralized-api/poc/validator_v1_test.go @@ -116,6 +116,10 @@ func TestValidationConfigDefaults(t *testing.T) { // fakeNodeClient satisfies mlnodeclient.MLNodeClient for testing. type fakeNodeClient struct{} +func (f fakeNodeClient) SetNodeState(ctx context.Context, state mlnodeclient.MLNodeState, errorReason string) error { + return nil +} + // failingNodeClient fails N times then succeeds, for testing retry logic. type failingNodeClient struct { mu sync.Mutex @@ -125,6 +129,10 @@ type failingNodeClient struct { validateErrs []error } +func (f *failingNodeClient) SetNodeState(ctx context.Context, state mlnodeclient.MLNodeState, errorReason string) error { + return nil +} + func newFailingNodeClient(maxFails int) *failingNodeClient { return &failingNodeClient{maxFails: maxFails} } diff --git a/fix plan.md b/fix plan.md new file mode 100644 index 000000000..ee57d078e --- /dev/null +++ b/fix plan.md @@ -0,0 +1,51 @@ + +### Phase 1: Make it Compile + +These must be done first — the project cannot build without them. + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P0-1 | Fix `fmtInt()` zero bug | `admin/onboarding_state_manager.go:99-108` | `fmtInt(0)` returns `""` instead of `"0"`. The loop `for n > 0` never executes when v==0. Fix: add `if v == 0 { return "0" }` or replace with `strconv.FormatInt(v, 10)` | +| P0-2 | Fix broken imports | `internal/event_listener/new_block_dispatcher.go` | Cherry-pick conflict left stale imports: `"decentralized-api/internal/poc"` (doesn't exist) and `"decentralized-api/poc"` (duplicate name). Remove stale imports, keep only the one actually used. | +| P0-3 | Fix MockClient interface | `mlnodeclient/mock.go` + `poc/validator_v1_test.go` | JF's last commit removed old PoC v1 methods, but main added new ones (e.g. `GenerateV2`). `MockClient` no longer satisfies `MLNodeClient` interface. Add stub implementations for all missing methods. Check `interface.go` for the full method list. Additionally, `poc/validator_v1_test.go` has two fake clients (`fakeNodeClient`, `failingNodeClient`) that also need a `SetNodeState` stub to satisfy the updated interface. | + +**Checkpoint: `go build ./...` should pass after Phase 1** + +--- + +### Phase 2: Fix Logic Bugs + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-1 | Fix `isTesting` always false | `admin/node_handlers.go:47-51` | `isTesting := false` is never set to `true`, so `MLNodeStatus()` never returns `TESTING` state. Users will never see "Running pre-PoC validation testing". Fix: add a `testingNodes map[string]bool` to `Server` struct, set it when auto-test starts, clear when finished. | +| P1-2 | Fix `testFailed` detection | `admin/node_handlers.go:48` | `testFailed := state.FailureReason != ""` treats ANY failure as test failure (could be PoC failure). Fix: check `state.MLNodeOnboardingState == string(MLNodeState_TEST_FAILED)` instead. | +| P1-4 | Add HTTP timeout | `broker/node_admin_commands.go:507` + `admin/mlnode_testing_orchestrator.go:121` | `http.Post()` uses default client with NO timeout — hangs forever if MLnode unresponsive. Fix: use `&http.Client{Timeout: 30 * time.Second}` | + +**Checkpoint: core logic should work correctly after Phase 2** + +--- + +### Phase 3: Consolidate & Clean Up + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-3 | Merge duplicated auto-test logic | `broker/node_admin_commands.go:432-528` + `admin/mlnode_testing_orchestrator.go:59-150` | Nearly identical ~100-line test logic exists in both files. Keep `MLnodeTestingOrchestrator` as single source of truth, remove `autoTestNodeIfTimeAllows()` from broker. Have broker call orchestrator instead. | +| P1-5 | Extract hardcoded block time | `apiconfig/constants.go` + 4 files | `6.0` seconds is hardcoded in 4 places. Define `const DefaultBlockTimeSeconds = 6.0` in `apiconfig/constants.go`, replace all occurrences. Files: `onboarding_state_manager.go:33`, `mlnode_testing_orchestrator.go:52`, `node_admin_commands.go:446`, `commands.go:71` | +| P1-6 | Extract hardcoded thresholds | `apiconfig/constants.go` + 4 files | Define `AutoTestMinSecondsBeforePoC = 3600` and `OnlineAlertLeadSeconds = 600`. Replace in: `node_admin_commands.go:448`, `mlnode_testing_orchestrator.go:56`, `status_reporter.go:44,23`, `onboarding_state_manager.go:34-35`, `commands.go:124` | +| P2-1 | Remove dead code | `admin/onboarding_state_manager.go:39-51` | Delete the commented-out old `MLNodeStatus` implementation | +| P2-2 | Wire up or remove `BuildNoModelGuidance()` | `admin/status_reporter.go:43-48` | Method exists but never called. Proposal requires showing "MLnode will be tested automatically when there is more than 1 hour until next PoC". Either call it in `getNodes` handler or remove. | + +**Checkpoint: no code duplication, no dead code, all constants unified** + +--- + +### Phase 4: Improve Robustness + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-7 | Suppress "no model" error at source | TBD (likely `broker/broker.go` reconciliation) | Proposal: "Don't show confusing error messages if participant isn't active yet". The `getNodes` handler avoids setting `userMsg` when inactive, but the original error log that prints "there is no model for ml node" still fires at the source. Find it and add inactive check. | +| P2-3 | Goroutine lifecycle for auto-tests | `broker/node_admin_commands.go` | Auto-tests run in fire-and-forget goroutines with no tracking or cancellation. Add `sync.WaitGroup` or channel-based test runner. | +| P2-4 | Stop silently ignoring errors | `broker/node_admin_commands.go` + `admin/node_handlers.go` | Multiple `_ = b.QueueMessage(cmd)` ignore failures. `IsParticipantActiveOnChain()` error ignored in `node_handlers.go:25-28`. At minimum, log all errors. | +| P2-6 | Add updateNode auto-test trigger | `admin/node_handlers.go` | `addNode()` has auto-test trigger (lines 242-269) but `updateNode()` doesn't. Add consistent behavior. | + +**Checkpoint: robust error handling, no goroutine leaks** From 43677e2bac640b3b5dc82d0786f7bad76b45a20a Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Sat, 28 Feb 2026 14:04:29 +0800 Subject: [PATCH 25/31] fix:Make it Compile --- .../event_listener/new_block_dispatcher.go | 2 +- .../server/admin/onboarding_state_manager.go | 21 ++++---- decentralized-api/mlnodeclient/mock.go | 30 +++++++++-- decentralized-api/poc/validator_v1_test.go | 8 +++ fix plan.md | 51 +++++++++++++++++++ 5 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 fix plan.md diff --git a/decentralized-api/internal/event_listener/new_block_dispatcher.go b/decentralized-api/internal/event_listener/new_block_dispatcher.go index 0786200cc..740a5c866 100644 --- a/decentralized-api/internal/event_listener/new_block_dispatcher.go +++ b/decentralized-api/internal/event_listener/new_block_dispatcher.go @@ -16,7 +16,7 @@ import ( "decentralized-api/cosmosclient" "decentralized-api/internal" "decentralized-api/internal/event_listener/chainevents" - "decentralized-api/internal/poc" + // "decentralized-api/internal/poc" // 移除无效导入 "decentralized-api/internal/seed" "decentralized-api/internal/server/admin" "decentralized-api/internal/validation" diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index 5972d9bcc..bd63795a4 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -97,13 +97,16 @@ func itoa(v int64) string { } func fmtInt(v int64) string { - var buf [20]byte - i := len(buf) - n := v - for n > 0 { - i-- - buf[i] = byte('0' + n%10) - n /= 10 - } - return string(buf[i:]) + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + n := v + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) } diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index 72d9ace96..ccdd90954 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -65,9 +65,9 @@ type MockClient struct { GetDiskSpaceCalled int // Capture parameters - LastInitDto *InitDto - LastInitValidateDto *InitDto - LastValidateBatch ProofBatch + LastInitDto *InitDtoV1 + LastInitValidateDto *InitDtoV1 + LastValidateBatch ProofBatchV1 LastInferenceModel string LastInferenceArgs []string LastTrainingParams struct { @@ -85,6 +85,30 @@ type MockClient struct { LastSetNodeReason string } +// Stub for missing MLNodeClient methods +func (m *MockClient) GetLoadedModels(ctx context.Context) ([]string, error) { + return []string{}, nil + } +func (m *MockClient) Reset() {} +func (m *MockClient) InitGenerateV1(ctx context.Context, dto InitDtoV1) error { return nil } +func (m *MockClient) InitValidateV1(ctx context.Context, dto InitDtoV1) error { return nil } +func (m *MockClient) ValidateBatchV1(ctx context.Context, batch ProofBatchV1) error { return nil } +func (m *MockClient) GetPowStatusV1(ctx context.Context) (*PowStatusResponseV1, error) { + return &PowStatusResponseV1{}, nil +} +func (m *MockClient) InitGenerateV2(ctx context.Context, req PoCInitGenerateRequestV2) (*PoCInitGenerateResponseV2, error) { + return &PoCInitGenerateResponseV2{}, nil +} +func (m *MockClient) GenerateV2(ctx context.Context, req PoCGenerateRequestV2) (*PoCGenerateResponseV2, error) { + return &PoCGenerateResponseV2{}, nil +} +func (m *MockClient) GetPowStatusV2(ctx context.Context) (*PoCStatusResponseV2, error) { + return &PoCStatusResponseV2{}, nil +} +func (m *MockClient) StopPowV2(ctx context.Context) (*PoCStopResponseV2, error) { + return &PoCStopResponseV2{}, nil +} + // NewMockClient creates a new mock client with default values func NewMockClient() *MockClient { return &MockClient{ diff --git a/decentralized-api/poc/validator_v1_test.go b/decentralized-api/poc/validator_v1_test.go index f7af79de1..6a10e01e1 100644 --- a/decentralized-api/poc/validator_v1_test.go +++ b/decentralized-api/poc/validator_v1_test.go @@ -116,6 +116,10 @@ func TestValidationConfigDefaults(t *testing.T) { // fakeNodeClient satisfies mlnodeclient.MLNodeClient for testing. type fakeNodeClient struct{} +func (f fakeNodeClient) SetNodeState(ctx context.Context, state mlnodeclient.MLNodeState, errorReason string) error { + return nil +} + // failingNodeClient fails N times then succeeds, for testing retry logic. type failingNodeClient struct { mu sync.Mutex @@ -125,6 +129,10 @@ type failingNodeClient struct { validateErrs []error } +func (f *failingNodeClient) SetNodeState(ctx context.Context, state mlnodeclient.MLNodeState, errorReason string) error { + return nil +} + func newFailingNodeClient(maxFails int) *failingNodeClient { return &failingNodeClient{maxFails: maxFails} } diff --git a/fix plan.md b/fix plan.md new file mode 100644 index 000000000..ee57d078e --- /dev/null +++ b/fix plan.md @@ -0,0 +1,51 @@ + +### Phase 1: Make it Compile + +These must be done first — the project cannot build without them. + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P0-1 | Fix `fmtInt()` zero bug | `admin/onboarding_state_manager.go:99-108` | `fmtInt(0)` returns `""` instead of `"0"`. The loop `for n > 0` never executes when v==0. Fix: add `if v == 0 { return "0" }` or replace with `strconv.FormatInt(v, 10)` | +| P0-2 | Fix broken imports | `internal/event_listener/new_block_dispatcher.go` | Cherry-pick conflict left stale imports: `"decentralized-api/internal/poc"` (doesn't exist) and `"decentralized-api/poc"` (duplicate name). Remove stale imports, keep only the one actually used. | +| P0-3 | Fix MockClient interface | `mlnodeclient/mock.go` + `poc/validator_v1_test.go` | JF's last commit removed old PoC v1 methods, but main added new ones (e.g. `GenerateV2`). `MockClient` no longer satisfies `MLNodeClient` interface. Add stub implementations for all missing methods. Check `interface.go` for the full method list. Additionally, `poc/validator_v1_test.go` has two fake clients (`fakeNodeClient`, `failingNodeClient`) that also need a `SetNodeState` stub to satisfy the updated interface. | + +**Checkpoint: `go build ./...` should pass after Phase 1** + +--- + +### Phase 2: Fix Logic Bugs + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-1 | Fix `isTesting` always false | `admin/node_handlers.go:47-51` | `isTesting := false` is never set to `true`, so `MLNodeStatus()` never returns `TESTING` state. Users will never see "Running pre-PoC validation testing". Fix: add a `testingNodes map[string]bool` to `Server` struct, set it when auto-test starts, clear when finished. | +| P1-2 | Fix `testFailed` detection | `admin/node_handlers.go:48` | `testFailed := state.FailureReason != ""` treats ANY failure as test failure (could be PoC failure). Fix: check `state.MLNodeOnboardingState == string(MLNodeState_TEST_FAILED)` instead. | +| P1-4 | Add HTTP timeout | `broker/node_admin_commands.go:507` + `admin/mlnode_testing_orchestrator.go:121` | `http.Post()` uses default client with NO timeout — hangs forever if MLnode unresponsive. Fix: use `&http.Client{Timeout: 30 * time.Second}` | + +**Checkpoint: core logic should work correctly after Phase 2** + +--- + +### Phase 3: Consolidate & Clean Up + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-3 | Merge duplicated auto-test logic | `broker/node_admin_commands.go:432-528` + `admin/mlnode_testing_orchestrator.go:59-150` | Nearly identical ~100-line test logic exists in both files. Keep `MLnodeTestingOrchestrator` as single source of truth, remove `autoTestNodeIfTimeAllows()` from broker. Have broker call orchestrator instead. | +| P1-5 | Extract hardcoded block time | `apiconfig/constants.go` + 4 files | `6.0` seconds is hardcoded in 4 places. Define `const DefaultBlockTimeSeconds = 6.0` in `apiconfig/constants.go`, replace all occurrences. Files: `onboarding_state_manager.go:33`, `mlnode_testing_orchestrator.go:52`, `node_admin_commands.go:446`, `commands.go:71` | +| P1-6 | Extract hardcoded thresholds | `apiconfig/constants.go` + 4 files | Define `AutoTestMinSecondsBeforePoC = 3600` and `OnlineAlertLeadSeconds = 600`. Replace in: `node_admin_commands.go:448`, `mlnode_testing_orchestrator.go:56`, `status_reporter.go:44,23`, `onboarding_state_manager.go:34-35`, `commands.go:124` | +| P2-1 | Remove dead code | `admin/onboarding_state_manager.go:39-51` | Delete the commented-out old `MLNodeStatus` implementation | +| P2-2 | Wire up or remove `BuildNoModelGuidance()` | `admin/status_reporter.go:43-48` | Method exists but never called. Proposal requires showing "MLnode will be tested automatically when there is more than 1 hour until next PoC". Either call it in `getNodes` handler or remove. | + +**Checkpoint: no code duplication, no dead code, all constants unified** + +--- + +### Phase 4: Improve Robustness + +| # | Task | File(s) | Description | +|---|------|---------|-------------| +| P1-7 | Suppress "no model" error at source | TBD (likely `broker/broker.go` reconciliation) | Proposal: "Don't show confusing error messages if participant isn't active yet". The `getNodes` handler avoids setting `userMsg` when inactive, but the original error log that prints "there is no model for ml node" still fires at the source. Find it and add inactive check. | +| P2-3 | Goroutine lifecycle for auto-tests | `broker/node_admin_commands.go` | Auto-tests run in fire-and-forget goroutines with no tracking or cancellation. Add `sync.WaitGroup` or channel-based test runner. | +| P2-4 | Stop silently ignoring errors | `broker/node_admin_commands.go` + `admin/node_handlers.go` | Multiple `_ = b.QueueMessage(cmd)` ignore failures. `IsParticipantActiveOnChain()` error ignored in `node_handlers.go:25-28`. At minimum, log all errors. | +| P2-6 | Add updateNode auto-test trigger | `admin/node_handlers.go` | `addNode()` has auto-test trigger (lines 242-269) but `updateNode()` doesn't. Add consistent behavior. | + +**Checkpoint: robust error handling, no goroutine leaks** From a5c7e3dfec493b78d62e3471671699ae716e954c Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Mon, 2 Mar 2026 10:59:17 +0800 Subject: [PATCH 26/31] fix: define missing mockClient variable --- .../broker/node_worker_commands_v1_test.go | 2 +- decentralized-api/mlnodeclient/mock.go | 95 +++++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/decentralized-api/broker/node_worker_commands_v1_test.go b/decentralized-api/broker/node_worker_commands_v1_test.go index 324f3a15a..06a1940c9 100644 --- a/decentralized-api/broker/node_worker_commands_v1_test.go +++ b/decentralized-api/broker/node_worker_commands_v1_test.go @@ -144,7 +144,7 @@ func TestInitValidateNodeCommandV1_Execute_AlreadyValidating(t *testing.T) { func TestInitValidateNodeCommandV1_Execute_TransitionsFromPOW(t *testing.T) { mockClient := mlnodeclient.NewMockClient() mockClient.CurrentState = mlnodeclient.MlNodeState_POW - mockClient.PowStatusV1 = mlnodeclient.PowStateV1Generating + mockClient.PowStatusV1 = mlnodeclient.PowStateV1Validating worker := &NodeWorker{ nodeId: "test-node", diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index ccdd90954..3f61a9387 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -44,6 +44,8 @@ type MockClient struct { DeleteModelError error ListModelsError error GetDiskSpaceError error + InitGenerateV1Error error + InitValidateV1Error error // Call tracking StopCalled int @@ -63,6 +65,15 @@ type MockClient struct { DeleteModelCalled int ListModelsCalled int GetDiskSpaceCalled int + InitGenerateV1Called int + InitValidateV1Called int + ValidateBatchV1Called int + GetPowStatusV1Called int + GetPowStatusV2Called int + StopPowV2Called int + InitGenerateV2Called int + PowStatusV1 PowStateV1 + SetV2Status func(status string) // Capture parameters LastInitDto *InitDtoV1 @@ -85,27 +96,99 @@ type MockClient struct { LastSetNodeReason string } +// Getter for StopCalled (for tests) +func (m *MockClient) GetStopCalled() int { + return m.StopCalled +} + // Stub for missing MLNodeClient methods func (m *MockClient) GetLoadedModels(ctx context.Context) ([]string, error) { - return []string{}, nil - } + return []string{}, nil +} func (m *MockClient) Reset() {} -func (m *MockClient) InitGenerateV1(ctx context.Context, dto InitDtoV1) error { return nil } -func (m *MockClient) InitValidateV1(ctx context.Context, dto InitDtoV1) error { return nil } -func (m *MockClient) ValidateBatchV1(ctx context.Context, batch ProofBatchV1) error { return nil } +func (m *MockClient) InitGenerateV1(ctx context.Context, dto InitDtoV1) error { + m.Mu.Lock() + defer m.Mu.Unlock() + + m.InitGenerateV1Called++ + m.LastInitDto = &dto + + if m.InitGenerateV1Error != nil { + return m.InitGenerateV1Error + } + + // V1: init generate should move node into POW + GENERATING + m.CurrentState = MlNodeState_POW + m.PowStatusV1 = PowStateV1Generating + return nil +} + +func (m *MockClient) InitValidateV1(ctx context.Context, dto InitDtoV1) error { + m.Mu.Lock() + defer m.Mu.Unlock() + + m.InitValidateV1Called++ + m.LastInitValidateDto = &dto + + if m.InitValidateV1Error != nil { + return m.InitValidateV1Error + } + + // V1: init validate should move node into POW + VALIDATING + m.CurrentState = MlNodeState_POW + m.PowStatusV1 = PowStateV1Validating + return nil +} + +func (m *MockClient) ValidateBatchV1(ctx context.Context, batch ProofBatchV1) error { + m.Mu.Lock() + defer m.Mu.Unlock() + + m.ValidateBatchV1Called++ + m.LastValidateBatch = batch + + // optional: validating a batch implies we are in validating mode + if m.PowStatusV1 == "" { + m.PowStatusV1 = PowStateV1Validating + } + return nil +} + func (m *MockClient) GetPowStatusV1(ctx context.Context) (*PowStatusResponseV1, error) { - return &PowStatusResponseV1{}, nil + m.Mu.Lock() + defer m.Mu.Unlock() + + m.GetPowStatusV1Called++ + + if m.GetPowStatusError != nil { + return nil, m.GetPowStatusError + } + + status := m.PowStatusV1 + if status == "" { + status = PowStateV1Stopped + if m.CurrentState == MlNodeState_POW { + status = PowStateV1Idle + } + } + return &PowStatusResponseV1{ + Status: status, + IsModelInitialized: false, + }, nil } func (m *MockClient) InitGenerateV2(ctx context.Context, req PoCInitGenerateRequestV2) (*PoCInitGenerateResponseV2, error) { + m.InitGenerateV2Called++ return &PoCInitGenerateResponseV2{}, nil } func (m *MockClient) GenerateV2(ctx context.Context, req PoCGenerateRequestV2) (*PoCGenerateResponseV2, error) { return &PoCGenerateResponseV2{}, nil } func (m *MockClient) GetPowStatusV2(ctx context.Context) (*PoCStatusResponseV2, error) { + m.GetPowStatusV2Called++ return &PoCStatusResponseV2{}, nil } func (m *MockClient) StopPowV2(ctx context.Context) (*PoCStopResponseV2, error) { + m.StopPowV2Called++ return &PoCStopResponseV2{}, nil } From ee2e2ed951bbb7590c56e8c6b696c9df67fd300f Mon Sep 17 00:00:00 2001 From: houjiaqi Date: Tue, 3 Mar 2026 11:43:08 +0800 Subject: [PATCH 27/31] fix: correct mock test bugs introduced --- .../broker/node_worker_commands_v1_test.go | 2 +- decentralized-api/mlnodeclient/mock.go | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/decentralized-api/broker/node_worker_commands_v1_test.go b/decentralized-api/broker/node_worker_commands_v1_test.go index 06a1940c9..324f3a15a 100644 --- a/decentralized-api/broker/node_worker_commands_v1_test.go +++ b/decentralized-api/broker/node_worker_commands_v1_test.go @@ -144,7 +144,7 @@ func TestInitValidateNodeCommandV1_Execute_AlreadyValidating(t *testing.T) { func TestInitValidateNodeCommandV1_Execute_TransitionsFromPOW(t *testing.T) { mockClient := mlnodeclient.NewMockClient() mockClient.CurrentState = mlnodeclient.MlNodeState_POW - mockClient.PowStatusV1 = mlnodeclient.PowStateV1Validating + mockClient.PowStatusV1 = mlnodeclient.PowStateV1Generating worker := &NodeWorker{ nodeId: "test-node", diff --git a/decentralized-api/mlnodeclient/mock.go b/decentralized-api/mlnodeclient/mock.go index 3f61a9387..8cd01caeb 100644 --- a/decentralized-api/mlnodeclient/mock.go +++ b/decentralized-api/mlnodeclient/mock.go @@ -73,7 +73,7 @@ type MockClient struct { StopPowV2Called int InitGenerateV2Called int PowStatusV1 PowStateV1 - SetV2Status func(status string) + V2Status string // Capture parameters LastInitDto *InitDtoV1 @@ -96,7 +96,14 @@ type MockClient struct { LastSetNodeReason string } -// Getter for StopCalled (for tests) +// SetV2Status sets the V2 PoC status for idempotency testing +func (m *MockClient) SetV2Status(status string) { + m.Mu.Lock() + defer m.Mu.Unlock() + m.V2Status = status +} + +// GetStopCalled returns the number of times Stop() was called (for tests) func (m *MockClient) GetStopCalled() int { return m.StopCalled } @@ -184,12 +191,16 @@ func (m *MockClient) GenerateV2(ctx context.Context, req PoCGenerateRequestV2) ( return &PoCGenerateResponseV2{}, nil } func (m *MockClient) GetPowStatusV2(ctx context.Context) (*PoCStatusResponseV2, error) { + m.Mu.Lock() + defer m.Mu.Unlock() m.GetPowStatusV2Called++ - return &PoCStatusResponseV2{}, nil + return &PoCStatusResponseV2{Status: m.V2Status}, nil } func (m *MockClient) StopPowV2(ctx context.Context) (*PoCStopResponseV2, error) { + m.Mu.Lock() + defer m.Mu.Unlock() m.StopPowV2Called++ - return &PoCStopResponseV2{}, nil + return &PoCStopResponseV2{Status: "OK", Results: []BackendResult{{Status: "stopped"}}}, nil } // NewMockClient creates a new mock client with default values From bbbecd68d197bc7c073203d9169ed5691d1242c9 Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Wed, 4 Mar 2026 10:54:15 +0800 Subject: [PATCH 28/31] consolidate autotest logic and unify timing constants --- decentralized-api/apiconfig/constants.go | 7 ++ decentralized-api/broker/commands.go | 6 +- .../broker/node_admin_commands.go | 110 +----------------- .../admin/mlnode_testing_orchestrator.go | 4 +- .../internal/server/admin/node_handlers.go | 34 ++++++ .../server/admin/onboarding_state_manager.go | 44 +++---- .../internal/server/admin/status_reporter.go | 5 +- 7 files changed, 66 insertions(+), 144 deletions(-) diff --git a/decentralized-api/apiconfig/constants.go b/decentralized-api/apiconfig/constants.go index 3e4d0a267..1e27c9e29 100644 --- a/decentralized-api/apiconfig/constants.go +++ b/decentralized-api/apiconfig/constants.go @@ -10,6 +10,13 @@ const ( MLNodeState_TEST_FAILED MLNodeOnboardingState = "TEST_FAILED" ) +// Timing constants used across broker/admin components +const ( + DefaultBlockTimeSeconds = 6.0 + AutoTestMinSecondsBeforePoC int64 = 3600 + OnlineAlertLeadSeconds int64 = 600 +) + // ParticipantState represents the state of a participant type ParticipantState string diff --git a/decentralized-api/broker/commands.go b/decentralized-api/broker/commands.go index 30ab176c2..81db6f9ad 100644 --- a/decentralized-api/broker/commands.go +++ b/decentralized-api/broker/commands.go @@ -67,8 +67,8 @@ func (c GetNodesCommand) Execute(b *Broker) { if blocksUntilNextPoC < 0 { blocksUntilNextPoC = 0 } - // Use default block time of 6 seconds - secondsUntilNextPoC = int64(float64(blocksUntilNextPoC) * 6.0) + // Use default block time constant + secondsUntilNextPoC = int64(float64(blocksUntilNextPoC) * apiconfig.DefaultBlockTimeSeconds) } nodeResponses := make([]NodeResponse, 0, len(b.nodes)) @@ -121,7 +121,7 @@ func (c GetNodesCommand) Execute(b *Broker) { State: stateCopy, }) if hasEpochInfo { - shouldOnline := currentPhase == types.PoCGeneratePhase || currentPhase == types.PoCGenerateWindDownPhase || currentPhase == types.PoCValidatePhase || currentPhase == types.PoCValidateWindDownPhase || secondsUntilNextPoC <= 600 + shouldOnline := currentPhase == types.PoCGeneratePhase || currentPhase == types.PoCGenerateWindDownPhase || currentPhase == types.PoCValidatePhase || currentPhase == types.PoCValidateWindDownPhase || secondsUntilNextPoC <= apiconfig.OnlineAlertLeadSeconds nodeResponses[len(nodeResponses)-1].State.Timing = &TimingInfo{ CurrentPhase: string(currentPhase), BlocksUntilNextPoC: blocksUntilNextPoC, diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index abfe1bd2b..1e8f64a4b 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -1,14 +1,9 @@ package broker import ( - "bytes" - "context" "decentralized-api/apiconfig" "decentralized-api/logging" - "decentralized-api/mlnodeclient" - "encoding/json" "fmt" - "net/http" "strings" "time" @@ -175,8 +170,7 @@ func (c RegisterNode) Execute(b *Broker) { logging.Info("RegisterNode. Registered node", types.Nodes, "node", c.Node) c.Response <- NodeCommandResponse{Node: &c.Node, Error: nil} - // Auto-test: if more than 1 hour until next PoC, perform basic validation - go b.autoTestNodeIfTimeAllows(node, "RegisterNode") + // Auto-test now handled by admin orchestrator } // UpdateNode updates an existing node's configuration while preserving runtime state @@ -275,8 +269,7 @@ func (c UpdateNode) Execute(b *Broker) { logging.Info("UpdateNode. Updated node configuration", types.Nodes, "node_id", c.Node.Id) c.Response <- NodeCommandResponse{Node: &c.Node, Error: nil} - // Auto-test: if more than 1 hour until next PoC, perform basic validation after config changes - go b.autoTestNodeIfTimeAllows(updated, "UpdateNode") + // Auto-test now handled by admin orchestrator } type RemoveNode struct { @@ -428,102 +421,3 @@ func (c SetNodeMLNodeOnboardingStateCommand) Execute(b *Broker) { c.Response <- true } - -// autoTestNodeIfTimeAllows performs a basic validation test for a node when -// there is more than 1 hour remaining until the next PoC. It loads each -// configured model and checks inference health, updating the node failure -// reason accordingly. The caller string is used for logging context. -func (b *Broker) autoTestNodeIfTimeAllows(node Node, caller string) { - epochState := b.phaseTracker.GetCurrentEpochState() - if epochState == nil || !epochState.IsSynced { - logging.Info(caller+". Auto-test skipped: epoch state unavailable", types.Nodes, "node_id", node.Id) - return - } - blocks := epochState.LatestEpoch.NextPoCStart() - epochState.CurrentBlock.Height - if blocks < 0 { - blocks = 0 - } - seconds := int64(float64(blocks) * 6.0) - logging.Info(caller+". Auto-test timing evaluated", types.Nodes, "node_id", node.Id, "seconds_until_next_poc", seconds) - if seconds <= 3600 { - logging.Info(caller+". Auto-test skipped: insufficient time", types.Nodes, "node_id", node.Id, "seconds_until_next_poc", seconds) - return - } - - version := b.configManager.GetCurrentNodeVersion() - client := b.mlNodeClientFactory.CreateClient(node.PoCUrlWithVersion(version), node.InferenceUrlWithVersion(version)) - logging.Info(caller+". Auto-test starting", types.Nodes, "node_id", node.Id, "models", len(node.Models)) - - // Helper function to handle test failures - setTestFailed := func(nodeId, errorReason string) { - cmd := NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) - _ = b.QueueMessage(cmd) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(nodeId, errorReason)) - - // Notify MLnode about the failure - notifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = client.SetNodeState(notifyCtx, mlnodeclient.MlNodeState_TEST_FAILED, errorReason) - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - for modelId, cfg := range node.Models { - if err := client.InferenceUp(ctx, modelId, cfg.Args); err != nil { - logging.Error(caller+". Auto-test model load failed", types.Nodes, "node_id", node.Id, "model", modelId, "error", err) - setTestFailed(node.Id, err.Error()) - return - } - } - - ok, err := client.InferenceHealth(ctx) - if err != nil || !ok { - reason := "health_not_ok" - if err != nil { - reason = err.Error() - } - logging.Error(caller+". Auto-test health check failed", types.Nodes, "node_id", node.Id, "error", reason) - setTestFailed(node.Id, reason) - return - } - - // Perform test inference request to validate response - firstModelId := getFirstModelIdFromNode(node) - if firstModelId != "" { - testRequest := map[string]interface{}{ - "model": firstModelId, - "messages": []map[string]string{{"role": "user", "content": "Hello, how are you?"}}, - "max_tokens": 10, - } - requestBody, err := json.Marshal(testRequest) - if err != nil { - logging.Error(caller+". Auto-test failed to create test request", types.Nodes, "node_id", node.Id, "error", err) - setTestFailed(node.Id, err.Error()) - return - } - - completionsUrl := node.InferenceUrlWithVersion(version) + "/v1/chat/completions" - client := &http.Client{Timeout: 30 * time.Second} - resp, err := client.Post(completionsUrl, "application/json", bytes.NewReader(requestBody)) - if err != nil { - logging.Error(caller+". Auto-test failed during inference request", types.Nodes, "node_id", node.Id, "error", err) - setTestFailed(node.Id, err.Error()) - return - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - reason := fmt.Sprintf("non_success_status_code: %d", resp.StatusCode) - logging.Error(caller+". Auto-test received non-success status code", types.Nodes, "node_id", node.Id, "status_code", resp.StatusCode, "error", reason) - setTestFailed(node.Id, reason) - return - } - } - - // On success, set node MLNodeOnboardingState to WAITING_FOR_POC - cmd := NewSetNodeMLNodeOnboardingStateCommand(node.Id, string(apiconfig.MLNodeState_WAITING_FOR_POC)) - _ = b.QueueMessage(cmd) - _ = b.QueueMessage(NewSetNodeFailureReasonCommand(node.Id, "")) - logging.Info(caller+". Auto-test passed", types.Nodes, "node_id", node.Id) -} diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index 539fa2638..db070caec 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -49,11 +49,11 @@ type MLnodeTestingOrchestrator struct { } func NewMLnodeTestingOrchestrator(cm *apiconfig.ConfigManager, nodeBroker *broker.Broker) *MLnodeTestingOrchestrator { - return &MLnodeTestingOrchestrator{configManager: cm, blockTimeSeconds: 6.0, nodeBroker: nodeBroker} + return &MLnodeTestingOrchestrator{configManager: cm, blockTimeSeconds: apiconfig.DefaultBlockTimeSeconds, nodeBroker: nodeBroker} } func (o *MLnodeTestingOrchestrator) ShouldAutoTest(secondsUntilNextPoC int64) bool { - return secondsUntilNextPoC > 3600 + return secondsUntilNextPoC > apiconfig.AutoTestMinSecondsBeforePoC } func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apiconfig.InferenceNodeConfig) *TestResult { diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 51f8d179d..82e696a25 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -58,6 +58,13 @@ func (s *Server) getNodes(ctx echo.Context) error { if participantActive { userMsg = sr.BuildMLNodeMessage(mlnodeState, secs, "") } + if guidance := sr.BuildNoModelGuidance(secs); guidance != "" { + if userMsg == "" { + userMsg = guidance + } else { + userMsg = userMsg + " " + guidance + } + } prevOnboarding := MLNodeOnboardingState(state.MLNodeOnboardingState) if prevOnboarding != mlnodeState { @@ -204,6 +211,33 @@ func (s *Server) createNewNode(ctx echo.Context) error { } // sync config file with updated node list syncNodesWithConfig(s.nodeBroker, s.configManager) + + // Auto-test trigger after update (uses orchestrator) + getCmd := broker.NewGetNodesCommand() + if err := s.nodeBroker.QueueMessage(getCmd); err == nil { + responses := <-getCmd.Response + var secs int64 + for _, resp := range responses { + if resp.Node.Id == newNode.Id && resp.State.Timing != nil { + secs = resp.State.Timing.SecondsUntilNextPoC + break + } + } + if s.tester.ShouldAutoTest(secs) { + s.statusReporter.LogTesting("Auto-testing MLnode configuration") + result := s.tester.RunNodeTest(context.Background(), *node) + if result != nil { + if result.Status == TestFailed { + cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) + _ = s.nodeBroker.QueueMessage(cmd) + } else { + cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, "") + _ = s.nodeBroker.QueueMessage(cmd) + } + s.latestTestResults[newNode.Id] = result + } + } + } return ctx.JSON(http.StatusOK, node) } else { node, err := s.addNode(newNode) diff --git a/decentralized-api/internal/server/admin/onboarding_state_manager.go b/decentralized-api/internal/server/admin/onboarding_state_manager.go index bd63795a4..5a282537a 100644 --- a/decentralized-api/internal/server/admin/onboarding_state_manager.go +++ b/decentralized-api/internal/server/admin/onboarding_state_manager.go @@ -30,26 +30,12 @@ type OnboardingStateManager struct { func NewOnboardingStateManager() *OnboardingStateManager { return &OnboardingStateManager{ timing: NewTimingCalculator(), - blockTimeSeconds: 6.0, - alertLeadSeconds: 600, - safeOfflineMinSec: 600, + blockTimeSeconds: apiconfig.DefaultBlockTimeSeconds, + alertLeadSeconds: apiconfig.OnlineAlertLeadSeconds, + safeOfflineMinSec: apiconfig.OnlineAlertLeadSeconds, } } -// func (m *OnboardingStateManager) MLNodeStatus(es *chainphase.EpochState, isTesting bool, testFailed bool) (MLNodeOnboardingState, string, bool) { -// if testFailed { -// return MLNodeState_TEST_FAILED, "Validation testing failed", true -// } -// if isTesting { -// return MLNodeState_TESTING, "Running pre-PoC validation testing", true -// } -// c := m.timing.Countdown(es, m.blockTimeSeconds, m.alertLeadSeconds) -// if c.ShouldBeOnline { -// return MLNodeState_WAITING_FOR_POC, "PoC starting soon (in " + formatShortDuration(c.NextPoCSeconds) + ") - MLnode must be online now", true -// } -// return MLNodeState_WAITING_FOR_POC, "Waiting for next PoC cycle (starts in " + formatShortDuration(c.NextPoCSeconds) + ") - you can safely turn off the server and restart it 10 minutes before PoC", false -// } - func (m *OnboardingStateManager) ParticipantStatus(isActive bool) ParticipantState { if isActive { return ParticipantState_ACTIVE_PARTICIPATING @@ -97,16 +83,16 @@ func itoa(v int64) string { } func fmtInt(v int64) string { - if v == 0 { - return "0" - } - var buf [20]byte - i := len(buf) - n := v - for n > 0 { - i-- - buf[i] = byte('0' + n%10) - n /= 10 - } - return string(buf[i:]) + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + n := v + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + return string(buf[i:]) } diff --git a/decentralized-api/internal/server/admin/status_reporter.go b/decentralized-api/internal/server/admin/status_reporter.go index a49963cd4..4cab3a629 100644 --- a/decentralized-api/internal/server/admin/status_reporter.go +++ b/decentralized-api/internal/server/admin/status_reporter.go @@ -1,6 +1,7 @@ package admin import ( + "decentralized-api/apiconfig" "decentralized-api/logging" "github.com/productscience/inference/x/inference/types" @@ -20,7 +21,7 @@ func (r *StatusReporter) BuildMLNodeMessage(state MLNodeOnboardingState, seconds } return "MLnode test failed: model '" + failingModel + "' could not be loaded" case MLNodeState_WAITING_FOR_POC: - if secondsUntilNextPoC <= 600 { + if secondsUntilNextPoC <= apiconfig.OnlineAlertLeadSeconds { return "PoC starting soon (in " + formatShortDuration(secondsUntilNextPoC) + ") - MLnode must be online now" } return "Waiting for next PoC cycle (starts in " + formatShortDuration(secondsUntilNextPoC) + ") - you can safely turn off the server and restart it 10 minutes before PoC" @@ -41,7 +42,7 @@ func (r *StatusReporter) BuildParticipantMessage(pstate ParticipantState) string } func (r *StatusReporter) BuildNoModelGuidance(secondsUntilNextPoC int64) string { - if secondsUntilNextPoC > 3600 { + if secondsUntilNextPoC > apiconfig.AutoTestMinSecondsBeforePoC { return "MLnode will be tested automatically when there is more than 1 hour until next PoC" } return "" From fa18f2c387926947a9f9ff045caa46ee29053cef Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Wed, 4 Mar 2026 16:59:00 +0800 Subject: [PATCH 29/31] fix: improve error handling and suppress confusing logs for inactive participants --- decentralized-api/broker/lock_helpers.go | 4 ++- .../broker/node_worker_commands.go | 9 +++++- .../admin/mlnode_testing_orchestrator.go | 16 ++++++++--- .../internal/server/admin/node_handlers.go | 28 ++++++++++++++----- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/decentralized-api/broker/lock_helpers.go b/decentralized-api/broker/lock_helpers.go index ac27bd89b..08db4ccb6 100644 --- a/decentralized-api/broker/lock_helpers.go +++ b/decentralized-api/broker/lock_helpers.go @@ -221,7 +221,9 @@ func DoWithLockedNodeHTTPRetry( } outcome = InferenceError{Message: msg} } - _ = b.QueueMessage(ReleaseNode{NodeId: node.Id, Outcome: outcome, Response: make(chan bool, 2)}) + if err := b.QueueMessage(ReleaseNode{NodeId: node.Id, Outcome: outcome, Response: make(chan bool, 2)}); err != nil { + logging.Warn("Failed to queue ReleaseNode message", types.Inferences, "node_id", node.Id, "error", err) + } if retry { if triggerRecheck { diff --git a/decentralized-api/broker/node_worker_commands.go b/decentralized-api/broker/node_worker_commands.go index 7e78fd265..89a5a6d23 100644 --- a/decentralized-api/broker/node_worker_commands.go +++ b/decentralized-api/broker/node_worker_commands.go @@ -134,10 +134,17 @@ func (c InferenceUpNodeCommand) Execute(ctx context.Context, worker *NodeWorker) } if !hasIntersection { + // Check if participant is active before reporting as error + // If participant not yet active, this is expected - use Info level + active, _ := worker.broker.IsParticipantActiveOnChain() + if !active { + logging.Info("No epoch models available for this node (participant not yet active)", types.Nodes, "node_id", worker.nodeId) + } else { + logging.Error("No epoch models available for this node", types.Nodes, "node_id", worker.nodeId) + } result.Succeeded = false result.Error = "No epoch models available for this node" result.FinalStatus = types.HardwareNodeStatus_FAILED - logging.Error(result.Error, types.Nodes, "node_id", worker.nodeId) return result } diff --git a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go index db070caec..12f5b2d9c 100644 --- a/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go +++ b/decentralized-api/internal/server/admin/mlnode_testing_orchestrator.go @@ -66,14 +66,20 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon setTestFailed := func(nodeId string, reason string) { if o.nodeBroker != nil { cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_TEST_FAILED)) - _ = o.nodeBroker.QueueMessage(cmd) - _ = o.nodeBroker.QueueMessage(broker.NewSetNodeFailureReasonCommand(nodeId, reason)) + if err := o.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to set MLnode onboarding state to TEST_FAILED", types.Nodes, "node_id", nodeId, "error", err) + } + if err := o.nodeBroker.QueueMessage(broker.NewSetNodeFailureReasonCommand(nodeId, reason)); err != nil { + logging.Warn("Failed to set node failure reason", types.Nodes, "node_id", nodeId, "error", err) + } } // Notify MLnode about the failure notifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _ = client.SetNodeState(notifyCtx, mlnodeclient.MlNodeState_TEST_FAILED, reason) + if err := client.SetNodeState(notifyCtx, mlnodeclient.MlNodeState_TEST_FAILED, reason); err != nil { + logging.Warn("Failed to notify MLnode about test failure", types.Nodes, "node_id", nodeId, "error", err) + } } metrics := TestMetrics{LoadMs: map[string]int64{}} @@ -139,7 +145,9 @@ func (o *MLnodeTestingOrchestrator) RunNodeTest(ctx context.Context, node apicon setTestSuccess := func(nodeId string) { if o.nodeBroker != nil { cmd := broker.NewSetNodeMLNodeOnboardingStateCommand(nodeId, string(apiconfig.MLNodeState_WAITING_FOR_POC)) - _ = o.nodeBroker.QueueMessage(cmd) + if err := o.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to set MLnode onboarding state to WAITING_FOR_POC", types.Nodes, "node_id", nodeId, "error", err) + } } } diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 82e696a25..10a5a9bb4 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -23,7 +23,9 @@ func (s *Server) getNodes(ctx echo.Context) error { chainActive := false if s.nodeBroker != nil { active, err := s.nodeBroker.IsParticipantActiveOnChain() - if err == nil { + if err != nil { + logging.Warn("Failed to check participant active status", types.Nodes, "error", err) + } else { chainActive = active } } @@ -229,10 +231,14 @@ func (s *Server) createNewNode(ctx echo.Context) error { if result != nil { if result.Status == TestFailed { cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to set node failure reason", types.Nodes, "node_id", newNode.Id, "error", err) + } } else { cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, "") - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to clear node failure reason", types.Nodes, "node_id", newNode.Id, "error", err) + } } s.latestTestResults[newNode.Id] = result } @@ -294,11 +300,15 @@ func (s *Server) addNode(newNode apiconfig.InferenceNodeConfig) (apiconfig.Infer if result != nil { if result.Status == TestFailed { cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to set node failure reason", types.Nodes, "node_id", newNode.Id, "error", err) + } } else { // Clear any previous failure reason on success cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, "") - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to clear node failure reason", types.Nodes, "node_id", newNode.Id, "error", err) + } } s.latestTestResults[newNode.Id] = result } @@ -333,10 +343,14 @@ func (s *Server) postNodeTest(ctx echo.Context) error { if result != nil { if result.Status == TestFailed { cmd := broker.NewSetNodeFailureReasonCommand(nodeId, result.Error) - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to set node failure reason", types.Nodes, "node_id", nodeId, "error", err) + } } else { cmd := broker.NewSetNodeFailureReasonCommand(nodeId, "") - _ = s.nodeBroker.QueueMessage(cmd) + if err := s.nodeBroker.QueueMessage(cmd); err != nil { + logging.Warn("Failed to clear node failure reason", types.Nodes, "node_id", nodeId, "error", err) + } } logging.Info("Admin manual test result", types.Nodes, "node_id", nodeId, "status", string(result.Status), "error", result.Error) s.latestTestResults[nodeId] = result From 9cada834f661743e56a1e38fceb4134f0580abe4 Mon Sep 17 00:00:00 2001 From: houjiaqi Date: Thu, 5 Mar 2026 11:20:46 +0800 Subject: [PATCH 30/31] fix: populate testingNodes, fix guidance concat, remove dead code --- .../broker/node_admin_commands.go | 7 --- .../internal/server/admin/node_handlers.go | 12 ++--- fix plan.md | 51 ------------------- 3 files changed, 6 insertions(+), 64 deletions(-) delete mode 100644 fix plan.md diff --git a/decentralized-api/broker/node_admin_commands.go b/decentralized-api/broker/node_admin_commands.go index 1e8f64a4b..94417bd1a 100644 --- a/decentralized-api/broker/node_admin_commands.go +++ b/decentralized-api/broker/node_admin_commands.go @@ -375,13 +375,6 @@ func (c UpdateNodeHardwareCommand) Execute(b *Broker) { c.Response <- nil } -func getFirstModelIdFromNode(node Node) string { - for modelId := range node.Models { - return modelId - } - return "" // Return empty if no models -} - // SetNodeMLNodeOnboardingStateCommand updates the MLNodeOnboardingState of a node type SetNodeMLNodeOnboardingStateCommand struct { NodeId string diff --git a/decentralized-api/internal/server/admin/node_handlers.go b/decentralized-api/internal/server/admin/node_handlers.go index 10a5a9bb4..2ee6778e0 100644 --- a/decentralized-api/internal/server/admin/node_handlers.go +++ b/decentralized-api/internal/server/admin/node_handlers.go @@ -60,12 +60,8 @@ func (s *Server) getNodes(ctx echo.Context) error { if participantActive { userMsg = sr.BuildMLNodeMessage(mlnodeState, secs, "") } - if guidance := sr.BuildNoModelGuidance(secs); guidance != "" { - if userMsg == "" { - userMsg = guidance - } else { - userMsg = userMsg + " " + guidance - } + if userMsg == "" { + userMsg = sr.BuildNoModelGuidance(secs) } prevOnboarding := MLNodeOnboardingState(state.MLNodeOnboardingState) @@ -227,7 +223,9 @@ func (s *Server) createNewNode(ctx echo.Context) error { } if s.tester.ShouldAutoTest(secs) { s.statusReporter.LogTesting("Auto-testing MLnode configuration") + s.testingNodes[newNode.Id] = true result := s.tester.RunNodeTest(context.Background(), *node) + delete(s.testingNodes, newNode.Id) if result != nil { if result.Status == TestFailed { cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) @@ -296,7 +294,9 @@ func (s *Server) addNode(newNode apiconfig.InferenceNodeConfig) (apiconfig.Infer } if s.tester.ShouldAutoTest(secs) { s.statusReporter.LogTesting("Auto-testing MLnode configuration") + s.testingNodes[newNode.Id] = true result := s.tester.RunNodeTest(context.Background(), *node) + delete(s.testingNodes, newNode.Id) if result != nil { if result.Status == TestFailed { cmd := broker.NewSetNodeFailureReasonCommand(newNode.Id, result.Error) diff --git a/fix plan.md b/fix plan.md deleted file mode 100644 index ee57d078e..000000000 --- a/fix plan.md +++ /dev/null @@ -1,51 +0,0 @@ - -### Phase 1: Make it Compile - -These must be done first — the project cannot build without them. - -| # | Task | File(s) | Description | -|---|------|---------|-------------| -| P0-1 | Fix `fmtInt()` zero bug | `admin/onboarding_state_manager.go:99-108` | `fmtInt(0)` returns `""` instead of `"0"`. The loop `for n > 0` never executes when v==0. Fix: add `if v == 0 { return "0" }` or replace with `strconv.FormatInt(v, 10)` | -| P0-2 | Fix broken imports | `internal/event_listener/new_block_dispatcher.go` | Cherry-pick conflict left stale imports: `"decentralized-api/internal/poc"` (doesn't exist) and `"decentralized-api/poc"` (duplicate name). Remove stale imports, keep only the one actually used. | -| P0-3 | Fix MockClient interface | `mlnodeclient/mock.go` + `poc/validator_v1_test.go` | JF's last commit removed old PoC v1 methods, but main added new ones (e.g. `GenerateV2`). `MockClient` no longer satisfies `MLNodeClient` interface. Add stub implementations for all missing methods. Check `interface.go` for the full method list. Additionally, `poc/validator_v1_test.go` has two fake clients (`fakeNodeClient`, `failingNodeClient`) that also need a `SetNodeState` stub to satisfy the updated interface. | - -**Checkpoint: `go build ./...` should pass after Phase 1** - ---- - -### Phase 2: Fix Logic Bugs - -| # | Task | File(s) | Description | -|---|------|---------|-------------| -| P1-1 | Fix `isTesting` always false | `admin/node_handlers.go:47-51` | `isTesting := false` is never set to `true`, so `MLNodeStatus()` never returns `TESTING` state. Users will never see "Running pre-PoC validation testing". Fix: add a `testingNodes map[string]bool` to `Server` struct, set it when auto-test starts, clear when finished. | -| P1-2 | Fix `testFailed` detection | `admin/node_handlers.go:48` | `testFailed := state.FailureReason != ""` treats ANY failure as test failure (could be PoC failure). Fix: check `state.MLNodeOnboardingState == string(MLNodeState_TEST_FAILED)` instead. | -| P1-4 | Add HTTP timeout | `broker/node_admin_commands.go:507` + `admin/mlnode_testing_orchestrator.go:121` | `http.Post()` uses default client with NO timeout — hangs forever if MLnode unresponsive. Fix: use `&http.Client{Timeout: 30 * time.Second}` | - -**Checkpoint: core logic should work correctly after Phase 2** - ---- - -### Phase 3: Consolidate & Clean Up - -| # | Task | File(s) | Description | -|---|------|---------|-------------| -| P1-3 | Merge duplicated auto-test logic | `broker/node_admin_commands.go:432-528` + `admin/mlnode_testing_orchestrator.go:59-150` | Nearly identical ~100-line test logic exists in both files. Keep `MLnodeTestingOrchestrator` as single source of truth, remove `autoTestNodeIfTimeAllows()` from broker. Have broker call orchestrator instead. | -| P1-5 | Extract hardcoded block time | `apiconfig/constants.go` + 4 files | `6.0` seconds is hardcoded in 4 places. Define `const DefaultBlockTimeSeconds = 6.0` in `apiconfig/constants.go`, replace all occurrences. Files: `onboarding_state_manager.go:33`, `mlnode_testing_orchestrator.go:52`, `node_admin_commands.go:446`, `commands.go:71` | -| P1-6 | Extract hardcoded thresholds | `apiconfig/constants.go` + 4 files | Define `AutoTestMinSecondsBeforePoC = 3600` and `OnlineAlertLeadSeconds = 600`. Replace in: `node_admin_commands.go:448`, `mlnode_testing_orchestrator.go:56`, `status_reporter.go:44,23`, `onboarding_state_manager.go:34-35`, `commands.go:124` | -| P2-1 | Remove dead code | `admin/onboarding_state_manager.go:39-51` | Delete the commented-out old `MLNodeStatus` implementation | -| P2-2 | Wire up or remove `BuildNoModelGuidance()` | `admin/status_reporter.go:43-48` | Method exists but never called. Proposal requires showing "MLnode will be tested automatically when there is more than 1 hour until next PoC". Either call it in `getNodes` handler or remove. | - -**Checkpoint: no code duplication, no dead code, all constants unified** - ---- - -### Phase 4: Improve Robustness - -| # | Task | File(s) | Description | -|---|------|---------|-------------| -| P1-7 | Suppress "no model" error at source | TBD (likely `broker/broker.go` reconciliation) | Proposal: "Don't show confusing error messages if participant isn't active yet". The `getNodes` handler avoids setting `userMsg` when inactive, but the original error log that prints "there is no model for ml node" still fires at the source. Find it and add inactive check. | -| P2-3 | Goroutine lifecycle for auto-tests | `broker/node_admin_commands.go` | Auto-tests run in fire-and-forget goroutines with no tracking or cancellation. Add `sync.WaitGroup` or channel-based test runner. | -| P2-4 | Stop silently ignoring errors | `broker/node_admin_commands.go` + `admin/node_handlers.go` | Multiple `_ = b.QueueMessage(cmd)` ignore failures. `IsParticipantActiveOnChain()` error ignored in `node_handlers.go:25-28`. At minimum, log all errors. | -| P2-6 | Add updateNode auto-test trigger | `admin/node_handlers.go` | `addNode()` has auto-test trigger (lines 242-269) but `updateNode()` doesn't. Add consistent behavior. | - -**Checkpoint: robust error handling, no goroutine leaks** From 7cdde5208bf01f3a30e68be3983700bd62d292b2 Mon Sep 17 00:00:00 2001 From: Ryanchen911 <22210860034@m.fudan.edu.cn> Date: Mon, 9 Mar 2026 16:05:53 +0800 Subject: [PATCH 31/31] test(event-listener): align TestRegularPocScenario with v2 inference idempotency --- decentralized-api/internal/event_listener/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/decentralized-api/internal/event_listener/integration_test.go b/decentralized-api/internal/event_listener/integration_test.go index a5e37cf2c..2ecb9e52a 100644 --- a/decentralized-api/internal/event_listener/integration_test.go +++ b/decentralized-api/internal/event_listener/integration_test.go @@ -650,8 +650,8 @@ func TestRegularPocScenario(t *testing.T) { require.NoError(t, err) waitForAsync(300 * time.Millisecond) - // After PoC validation ends, nodes return to inference (+1 stop for inference transition) - expected = NodeClientAssertion{StopCalled: 2, InitGenerateV2Called: 1, InitValidateCalled: 0, InferenceUpCalled: 2} + // After PoC validation ends, V2 nodes remain in inference (PoC runs inside vLLM), no extra Stop/InferenceUp needed + expected = NodeClientAssertion{StopCalled: 1, InitGenerateV2Called: 1, InitValidateCalled: 0, InferenceUpCalled: 1} assertNodeClient(t, expected, node1Client) assertNodeClient(t, expected, node2Client) setup.assertNode("node-1", func(n broker.NodeResponse) {