Skip to content

gui, topo: allow_crash tile flag #8449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jherrera-jump wants to merge 1 commit into
base: main
Choose a base branch
from
Open

gui, topo: allow_crash tile flag #8449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions src/app/firedancer/topology.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1299,41 +1299,41 @@ fd_topo_initialize( config_t * config ) {
fd_topob_wksp( topo, "gui" );
fd_topob_wksp( topo, "gui_replay" );

/**/ fd_topob_tile( topo, "gui", "gui", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1, 0 );
/**/ fd_topob_tile( topo, "gui", "gui", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1, 0 )->allow_crash = 1;

/* Read banks */
/**/ fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "gui", 0UL ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
/**/ fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "gui", 0UL ) ], banks_locks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

/* release ownership of banks */
/**/ fd_topob_link( topo, "gui_replay", "gui_replay", 128, 0UL,2UL ); /* burst==2 since a bank and its parent may be sent in one burst */
/**/ fd_topob_link( topo, "gui_replay", "gui_replay", 128, 0UL, 1UL );

/**/ fd_topob_tile_out( topo, "gui", 0UL, "gui_replay", 0UL );
/**/ fd_topob_tile_in ( topo, "replay", 0UL, "metric_in", "gui_replay", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );

/* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */
FOR(net_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "net_gossvf", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "repair_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "shred_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "gossip_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "replay_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "genesi_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */
FOR(net_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "net_gossvf", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "repair_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "shred_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "gossip_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "replay_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "genesi_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
if( leader_enabled ) {
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "pack_poh", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "pack_execle", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(execle_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "execle_poh", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "pack_poh", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "pack_execle", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(execle_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "execle_poh", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
}
FOR(execrp_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "execrp_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(execrp_tile_cnt) fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "execrp_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );

if( FD_LIKELY( snapshots_enabled ) ) {
/**/ fd_topob_tile_in ( topo, "gui", 0UL, "metric_in", "snapct_gui", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in ( topo, "gui", 0UL, "metric_in", "snapin_gui", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "snapct_gui", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "snapin_gui", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
}
if( FD_UNLIKELY( config->tiles.bundle.enabled ) ) {
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "bundle_status", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "bundle_status", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
}
}

Expand Down
66 changes: 51 additions & 15 deletions src/app/shared/commands/run/run.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ run_cmd_perm( args_t * args,
}

struct pidns_clone_args {
config_t const * config;
int * pipefd;
int closefd;
config_t * config;
int * pipefd;
int closefd;
};

extern char fd_log_private_path[ 1024 ]; /* empty string on start */
Expand Down Expand Up @@ -231,7 +231,7 @@ main_pid_namespace( void * _args ) {
if( FD_UNLIKELY( close( args->closefd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
}

config_t const * config = args->config;
config_t * config = args->config;

fd_log_thread_set( "pidns" );
ulong pid = fd_sandbox_getpid(); /* Need to read /proc again.. we got a new PID from clone */
Expand Down Expand Up @@ -441,18 +441,43 @@ main_pid_namespace( void * _args ) {

char * tile_name = child_names[ i ];
ulong tile_idx = child_idxs[ i ];
ulong tile_id = config->topo.tiles[ tile_idx ].kind_id;
if( FD_UNLIKELY( tile_idx==ULONG_MAX ) ) FD_LOG_ERR(( "unreachable" ));
fd_topo_tile_t const * tile = &config->topo.tiles[ tile_idx ];

if( FD_UNLIKELY( !WIFEXITED( wstatus ) ) ) {
FD_LOG_ERR_NOEXIT(( "tile %s:%lu exited with signal %d (%s)", tile_name, tile_id, WTERMSIG( wstatus ), fd_io_strsignal( WTERMSIG( wstatus ) ) ));
fd_sys_util_exit_group( WTERMSIG( wstatus ) ? WTERMSIG( wstatus ) : 1 );
if( FD_UNLIKELY( tile->allow_crash ) ) {
FD_LOG_ERR_NOEXIT(( "expendable tile %s:%lu crashed with signal %d (%s)", tile_name, tile->kind_id, WTERMSIG( wstatus ), fd_io_strsignal( WTERMSIG( wstatus ) ) ));

/* We need to make all reliable links unreliable by
provisioning infinite credits to prevent the application
from stalling. */
for( ulong j=0UL; j<tile->in_cnt; j++ ) {
if( FD_UNLIKELY( !tile->in_link_poll[ j ] || !tile->in_link_reliable[ j ] ) ) continue;

ulong fseq_id = tile->in_link_fseq_obj_id[ j ];
fd_topo_wksp_t * wksp = &config->topo.workspaces[ config->topo.objs[ fseq_id ].wksp_id ];
fd_topo_join_workspace( &config->topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, 0 );
ulong * fseq = fd_fseq_join( fd_topo_obj_laddr( &config->topo, fseq_id ) );
if( FD_UNLIKELY( !fseq ) ) {
FD_LOG_ERR_NOEXIT(( "failed to join fseq" ));
fd_sys_util_exit_group( 1 );
}
fd_fseq_update( fseq, ULONG_MAX-1UL );
fd_topo_leave_workspace( &config->topo, wksp );

fd_topo_link_t * in_link = &config->topo.links[ tile->in_link_id[ j ] ];
FD_LOG_NOTICE(( "demoted reliable in-link %s(%lu)->%s to unreliable", in_link->name, in_link->kind_id, tile->name ));
}
} else {
FD_LOG_ERR_NOEXIT(( "tile %s:%lu crashed with signal %d (%s)", tile_name, tile->kind_id, WTERMSIG( wstatus ), fd_io_strsignal( WTERMSIG( wstatus ) ) ));
fd_sys_util_exit_group( WTERMSIG( wstatus ) ? WTERMSIG( wstatus ) : 1 );
}
} else {
int exit_code = WEXITSTATUS( wstatus );
if( FD_LIKELY( !exit_code && tile_idx!=ULONG_MAX && config->topo.tiles[ tile_idx ].allow_shutdown ) ) {
found = 1;
FD_LOG_INFO(( "tile %s:%lu exited gracefully with code %d", tile_name, tile_id, exit_code ));
if( FD_LIKELY( tile->allow_shutdown ) ) {
FD_LOG_INFO(( "tile %s:%lu exited gracefully with code %d", tile_name, tile->kind_id, exit_code ));
} else {
FD_LOG_ERR_NOEXIT(( "tile %s:%lu exited with code %d", tile_name, tile_id, exit_code ));
FD_LOG_ERR_NOEXIT(( "tile %s:%lu exited unexpectedly with code %d", tile_name, tile->kind_id, exit_code ));
fd_sys_util_exit_group( exit_code ? exit_code : 1 );
}
}
Expand All @@ -465,9 +490,9 @@ main_pid_namespace( void * _args ) {
}

int
clone_firedancer( config_t const * config,
int close_fd,
int * out_pipe ) {
clone_firedancer( config_t * config,
int close_fd,
int * out_pipe ) {
/* This pipe is here just so that the child process knows when the
parent has died (it will get a HUP). */
int pipefd[2];
Expand Down Expand Up @@ -633,7 +658,18 @@ initialize_workspaces( config_t * config ) {
}
fd_topo_join_workspace( &config->topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, 0 );
fd_topo_wksp_new( &config->topo, wksp, CALLBACKS );
fd_topo_leave_workspace( &config->topo, wksp );

/* Don't unmap fseq wksp for tile with allow_crash=1, since we need
to retain access in order to demote reliable links to unreliable. */
int is_crashable_fseq_wksp = 0;
for( ulong j=0UL; j<config->topo.tile_cnt; j++ ){
fd_topo_tile_t * tile = &config->topo.tiles[ j ];
for( ulong k=0UL; k<tile->in_cnt; k++ ) {
if( FD_UNLIKELY( !tile->allow_crash || !tile->in_link_poll[ k ] || !tile->in_link_reliable[ k ] ) ) continue;
if( FD_UNLIKELY( i==config->topo.objs[ tile->in_link_fseq_obj_id[ k ] ].wksp_id ) ) is_crashable_fseq_wksp = 1;
}
}
if( FD_LIKELY( !is_crashable_fseq_wksp ) ) fd_topo_leave_workspace( &config->topo, wksp );
}

if( FD_UNLIKELY( seteuid( uid ) ) ) FD_LOG_ERR(( "seteuid() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
Expand Down
6 changes: 3 additions & 3 deletions src/app/shared/commands/run/run.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ void *
create_clone_stack( void );

int
clone_firedancer( config_t const * config,
int close_fd,
int * out_pipe );
clone_firedancer( config_t * config,
int close_fd,
int * out_pipe );

void
fdctl_check_configure( config_t const * config );
Expand Down
44 changes: 20 additions & 24 deletions src/disco/gui/fd_gui.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2830,28 +2830,24 @@ fd_gui_handle_tower_update( fd_gui_t * gui,
}

void
fd_gui_handle_replay_update( fd_gui_t * gui,
fd_gui_slot_completed_t * slot_completed,
fd_hash_t const * block_hash,
ulong vote_slot,
ulong storage_slot,
ulong rooted_slot,
ulong identity_balance,
long now ) {
fd_gui_handle_replay_update( fd_gui_t * gui,
fd_replay_slot_completed_t const * slot_completed,
ulong vote_slot,
long now ) {
(void)now;

if( FD_LIKELY( rooted_slot!=ULONG_MAX && gui->summary.slot_rooted!=rooted_slot ) ) {
fd_gui_handle_rooted_slot( gui, rooted_slot );
if( FD_LIKELY( slot_completed->root_slot!=ULONG_MAX && gui->summary.slot_rooted!=slot_completed->root_slot ) ) {
fd_gui_handle_rooted_slot( gui, slot_completed->root_slot );
}

if( FD_LIKELY( gui->summary.slot_storage!=storage_slot ) ) {
gui->summary.slot_storage = storage_slot;
if( FD_LIKELY( gui->summary.slot_storage!=slot_completed->storage_slot ) ) {
gui->summary.slot_storage = slot_completed->storage_slot;
fd_gui_printf_storage_slot( gui );
fd_http_server_ws_broadcast( gui->http );
}

if( FD_UNLIKELY( identity_balance!=ULONG_MAX && gui->summary.identity_account_balance!=identity_balance ) ) {
gui->summary.identity_account_balance = identity_balance;
if( FD_UNLIKELY( slot_completed->identity_balance!=ULONG_MAX && gui->summary.identity_account_balance!=slot_completed->identity_balance ) ) {
gui->summary.identity_account_balance = slot_completed->identity_balance;

fd_gui_printf_identity_balance( gui );
fd_http_server_ws_broadcast( gui->http );
Expand All @@ -2873,12 +2869,12 @@ fd_gui_handle_replay_update( fd_gui_t * gui,

if( FD_UNLIKELY( slot->mine ) ) {
fd_gui_leader_slot_t * lslot = fd_gui_get_leader_slot( gui, slot->slot );
if( FD_LIKELY( lslot ) ) fd_memcpy( lslot->block_hash.uc, block_hash->uc, sizeof(fd_hash_t) );
if( FD_LIKELY( lslot ) ) fd_memcpy( lslot->block_hash.uc, slot_completed->block_hash.uc, sizeof(fd_hash_t) );
}

slot->completed_time = slot_completed->completed_time;
slot->completed_time = slot_completed->completion_time_nanos;
slot->parent_slot = slot_completed->parent_slot;
slot->max_compute_units = fd_uint_if( slot_completed->max_compute_units==UINT_MAX, slot->max_compute_units, slot_completed->max_compute_units );
slot->max_compute_units = fd_uint_if( slot_completed->cost_tracker.block_cost_limit==ULONG_MAX, slot->max_compute_units, (uint)slot_completed->cost_tracker.block_cost_limit );
if( FD_LIKELY( slot->level<FD_GUI_SLOT_LEVEL_COMPLETED ) ) {
/* Typically a slot goes from INCOMPLETE to COMPLETED but it can
happen that it starts higher. One such case is when we
Expand All @@ -2893,15 +2889,15 @@ fd_gui_handle_replay_update( fd_gui_t * gui,
slot->level = FD_GUI_SLOT_LEVEL_COMPLETED;
}
}
slot->total_txn_cnt = slot_completed->total_txn_cnt;
slot->vote_txn_cnt = slot_completed->vote_txn_cnt;
slot->failed_txn_cnt = slot_completed->failed_txn_cnt;
slot->nonvote_failed_txn_cnt = slot_completed->nonvote_failed_txn_cnt;
slot->total_txn_cnt = fd_uint_if( slot_completed->total_txn_cnt==ULONG_MAX, slot->total_txn_cnt, (uint)slot_completed->total_txn_cnt );
slot->vote_txn_cnt = fd_uint_if( slot_completed->vote_txn_cnt==ULONG_MAX, slot->vote_txn_cnt, (uint)slot_completed->vote_txn_cnt );
slot->failed_txn_cnt = fd_uint_if( slot_completed->failed_txn_cnt==ULONG_MAX, slot->failed_txn_cnt, (uint)slot_completed->failed_txn_cnt );
slot->nonvote_failed_txn_cnt = fd_uint_if( slot_completed->nonvote_failed_txn_cnt==ULONG_MAX, slot->nonvote_failed_txn_cnt, (uint)slot_completed->nonvote_failed_txn_cnt );
slot->transaction_fee = slot_completed->transaction_fee;
slot->priority_fee = slot_completed->priority_fee;
slot->tips = slot_completed->tips;
slot->compute_units = slot_completed->compute_units;
slot->shred_cnt = slot_completed->shred_cnt;
slot->compute_units = fd_uint_if( slot_completed->cost_tracker.block_cost==ULONG_MAX, slot->compute_units, (uint)slot_completed->cost_tracker.block_cost );
slot->shred_cnt = fd_uint_if( slot_completed->shred_cnt==ULONG_MAX, slot->shred_cnt, (uint)slot_completed->shred_cnt );
slot->vote_slot = vote_slot;

try_publish_vote_status( gui, slot_completed->slot );
Expand Down Expand Up @@ -2935,7 +2931,7 @@ fd_gui_handle_replay_update( fd_gui_t * gui,
fd_gui_slot_staged_shred_event_t * slot_complete_event = &gui->shreds.staged[ gui->shreds.staged_tail % FD_GUI_SHREDS_STAGING_SZ ];
gui->shreds.staged_tail++;
slot_complete_event->event = FD_GUI_SLOT_SHRED_SHRED_SLOT_COMPLETE;
slot_complete_event->timestamp = slot_completed->completed_time;
slot_complete_event->timestamp = slot_completed->completion_time_nanos;
slot_complete_event->shred_idx = USHORT_MAX;
slot_complete_event->slot = slot->slot;

Expand Down
Loading
Loading