diff --git a/config/everything.mk b/config/everything.mk index e6c1f03796b..b9e6efe792b 100644 --- a/config/everything.mk +++ b/config/everything.mk @@ -232,17 +232,6 @@ $(OBJDIR)/unit-test/automatic.txt: $(LOCAL_MKS) $(RM) $@ @$(foreach test,$(RUN_UNIT_TEST),echo $(test)>>$@;) -# Generate list of automatic integration tests from $(call run-integration-test,...) -integration-test: $(OBJDIR)/integration-test/automatic.txt -define _run-integration-test -RUN_INTEGRATION_TEST+=$(OBJDIR)/integration-test/$(1) -endef -$(OBJDIR)/integration-test/automatic.txt: $(LOCAL_MKS) - $(MKDIR) "$(OBJDIR)/integration-test" - $(RM) $@ - @$(foreach test,$(RUN_INTEGRATION_TEST),echo $(test)>>$@;) - $(TOUCH) "$@" - ifndef FD_HAS_FUZZ FUZZ_EXTRA:=$(OBJDIR)/lib/libfd_fuzz_stub.a endif @@ -279,7 +268,6 @@ make-shared = $(eval $(call _make-exe,$(1),$(2),$(3),lib,lib,$(4) $(LDFLAGS_S make-unit-test = $(eval $(call _make-exe,$(1),$(2),$(3),unit-test,unit-test,$(4) $(LDFLAGS_EXE))) run-unit-test = $(eval $(call _run-unit-test,$(1))) make-integration-test = $(eval $(call _make-exe,$(1),$(2),$(3),integration-test,integration-test,$(4) $(LDFLAGS_EXE))) -run-integration-test = $(eval $(call _run-integration-test,$(1))) make-fuzz-test = $(eval $(call _fuzz-test,$(1),$(2),$(3),$(4) $(LDFLAGS_EXE))) ############################## diff --git a/contrib/offline-replay/offline_replay.toml b/contrib/offline-replay/offline_replay.toml index fb7a8752ec3..ff2ebf84eb2 100644 --- a/contrib/offline-replay/offline_replay.toml +++ b/contrib/offline-replay/offline_replay.toml @@ -15,6 +15,7 @@ [accounts] file_size_gib = {funk_pages} max_accounts = {index_max} + in_memory_only = true [runtime] max_live_slots = 128 max_fork_width = 4 diff --git a/contrib/test/run_fd_shred_cap.sh b/contrib/test/run_fd_shred_cap.sh index 30eef3b4178..f4ad27d6229 100755 --- a/contrib/test/run_fd_shred_cap.sh +++ b/contrib/test/run_fd_shred_cap.sh @@ -84,6 +84,7 @@ echo " [layout] execrp_tile_count = 8 [accounts] + in_memory_only = true max_accounts = 150000000 file_size_gib = 100 [tiles] diff --git a/contrib/test/run_integration_tests.sh b/contrib/test/run_integration_tests.sh index 9c50d098048..11728dbe862 100755 --- a/contrib/test/run_integration_tests.sh +++ b/contrib/test/run_integration_tests.sh @@ -12,29 +12,6 @@ set -eou pipefail TESTS= VERBOSE=0 -# Read command-line args - -while [[ $# -gt 0 ]]; do - - FLAG="$1" - shift 1 - - case "$FLAG" in - "--tests") - TESTS="$1" - shift 1 - ;; - "-v") - VERBOSE=1 - ;; - *) - echo "Unknown flag: $FLAG" >&2 - exit 1 - ;; - esac - -done - if [[ -z "$TESTS" && -z "${RECURSE_GUARD:-}" ]]; then # No tests given, so indirect test execution through Make and retry. # This ensures that we select the correct build dir for the current @@ -51,31 +28,6 @@ AVAILABLE_JOBS=1 # Clean up process tree on exit trap 'exit' INT QUIT TERM -# Track list of PIDs of child processes -declare -A PIDS=() -# Remember unit name of each PID -declare -A PID2UNIT=() -# Remember logfile name of each PID -declare -A PID2LOG=() - -# Read in list of automatic integration tests to schedule as jobs -declare -a TEST_LIST=() - -if [[ -s "$TESTS" ]]; then - while read -r line; do - if [[ "$line" =~ ^[[:space:]]*# ]]; then - continue - fi - TEST_LIST+=( "$line" ) - done < <(grep -v '^#' "$TESTS") -fi - -echo "test.sh: Scheduling ${#TEST_LIST[@]} tests in sequence" >&2 - -rc_path () { - echo "/tmp/.pid-$1.rc" -} - runner () { local pid="$BASHPID" local prog="$1" @@ -116,14 +68,15 @@ runner () { } > "$rcpath" } -# dispatch numa_idx cpu_idx cmdline... -# Fork task dispatch () { - # Craft command line args local prog="$1" local progname="${prog##*/}" shift 1 + if [[ ! -f "$prog" ]]; then + return 0 + fi + # Create log dir local logdir logdir="$(dirname "$(dirname "$prog")")/log/$progname" @@ -135,82 +88,8 @@ dispatch () { fi # Dispatch - runner "$prog" "$log" "$@" & - local pid="$!" - - # Remember PID - PIDS["$pid"]=$pid - PID2UNIT["$pid"]="$progname" - PID2LOG["$pid"]="$log" -} - -# sow -# schedule tasks until max concurrency reached -sow () { - if [[ "${#TEST_LIST[@]}" -eq 0 ]]; then return; fi - if [[ "${AVAILABLE_JOBS}" -eq 0 ]]; then return; fi - - # Found a free CPU! - local test="${TEST_LIST[0]}" - TEST_LIST=( "${TEST_LIST[@]:1}" ) - AVAILABLE_JOBS="$(( AVAILABLE_JOBS - 1 ))" - dispatch "$test" + runner "$prog" "$log" "$@" } -FAIL_CNT=0 - -# reap -# wait for a job to finish -reap () { - wait -n "${PIDS[@]}" - # Clean up finished jobs - for pid in "${PIDS[@]}"; do - if [[ ! -d "/proc/$pid" ]]; then - # Job finished - local rcfile; rcfile="$(rc_path "$pid")" - local rc - local elapsed - { - IFS= read -r rc - IFS= read -r elapsed - } < "$rcfile" - local unit="${PID2UNIT["$pid"]}" - local log="${PID2LOG["$pid"]}" - local logfull="${log%.log}-full.log" - unset PIDS["$pid"] - unset PID2UNIT["$pid"] - unset PID2LOG["$pid"] - AVAILABLE_JOBS="$(( AVAILABLE_JOBS + 1 ))" - if [[ "$rc" -ne 0 ]]; then - FAIL_CNT="$(( FAIL_CNT + 1 ))" - printf "\033[0;31mFAIL\033[0m%12s %s (exit %d): %s\n" "$elapsed" "$unit" "$rc" "$logfull" >&2 - grep -sv "Log at" "$logfull" | sed -e "$(printf "s/^/%19s%-20s /" '' "$unit")" || true >&2 - else - printf "\033[0;32mOK \033[0m%12s %s\n" "$elapsed" "$unit" >&2 - fi - fi - done -} - -while [[ "${#TEST_LIST[@]}" -gt 0 ]]; do - sow - reap -done -while [[ "${#PIDS[@]}" -gt 0 ]]; do - reap -done - -if [[ "$FAIL_CNT" -gt 0 ]]; then - echo -e "\033[0;31mFAIL\033[0m ($FAIL_CNT failure)" >&2 - exit 1 -else - echo -e "\033[0;32mPASS\033[0m" >&2 - exit 0 -fi - -# TODO add firedancer-dev integration tests here - -# Broken because genesis creation is unreliable -#src/app/firedancer-dev/tests/test_single_transfer.sh -# Broken because 'firedancer-dev txn' is unreliable -#src/app/firedancer-dev/tests/test_single_txn.sh +dispatch "$OBJDIR"/unit-test/test_fddev +dispatch "$OBJDIR"/unit-test/test_firedancer_dev --testnet diff --git a/contrib/test/run_solcap_tests.sh b/contrib/test/run_solcap_tests.sh index 67948113f5d..6474636bdfc 100755 --- a/contrib/test/run_solcap_tests.sh +++ b/contrib/test/run_solcap_tests.sh @@ -92,6 +92,7 @@ cat > "$DUMP/$LEDGER/mainnet-376969880_current.toml" << EOF [tiles.rpc] enabled = false [accounts] + in_memory_only = true file_size_gib = 1 max_accounts = 2000000 [runtime] diff --git a/contrib/test/test_firedancer_leader.sh b/contrib/test/test_firedancer_leader.sh deleted file mode 100755 index 697eae3ad7e..00000000000 --- a/contrib/test/test_firedancer_leader.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -IFS=$'\n\t' -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -FD_DIR="$SCRIPT_DIR/../.." -OBJDIR=${OBJDIR:-build/native/${CC}} -AGAVE_PATH=${AGAVE_PATH:='./agave/target/release'} - -cd ../test-ledger/ - -cleanup() { - sudo killall -9 -q firedancer-dev || true -# $FD_DIR/$OBJDIR/bin/firedancer-dev configure fini all --config firedancer-dev.toml -} -trap cleanup EXIT SIGINT SIGTERM - -sudo killall -9 -q firedancer-dev || true - -# if fd_frank_ledger is not on path then use the one in the home directory -if ! command -v firedancer-dev > /dev/null; then - PATH="$FD_DIR/$OBJDIR/bin":$PATH -fi - -_PRIMARY_INTERFACE=$(ip route show default | awk '/default/ {print $5}') -PRIMARY_IP=$(ip addr show $_PRIMARY_INTERFACE | awk '/inet / {print $2}' | cut -d/ -f1 | head -n1) - -while [ $($AGAVE_PATH/solana -u localhost epoch-info --output json | jq .blockHeight) -le 150 ]; do - sleep 1 -done - -FULL_SNAPSHOT=$(wget -c -nc -S --trust-server-names http://$PRIMARY_IP:8899/snapshot.tar.bz2 |& grep 'location:' | cut -d/ -f2) -SHRED_VERS=`grep shred_version: validator.log | sed -e 's@.*shred_version: \([0-9]*\).*@\1@'` - -sudo rm -f /tmp/localnet.funk -sudo rm -f /tmp/localnet.blockstore -sudo rm -f firedancer-dev.log - -echo " -[layout] - verify_tile_count = 16 -[gossip] - entrypoints = [\"$PRIMARY_IP:8001\"] - port = 8700 -[tiles] - [tiles.repair] - repair_intake_listen_port = 8701 - repair_serve_listen_port = 8702 - [tiles.replay] - capture = \"firedancer-dev.solcap\" - snapshot = \"$FULL_SNAPSHOT\" - [tiles.gui] - enabled = false - gui_listen_address = \"64.130.51.169\" - gui_listen_port = 8080 -[consensus] - expected_shred_version = $SHRED_VERS - vote = true -[paths] - identity_key = \"fd-identity-keypair.json\" - vote_account = \"fd-vote-keypair.json\" -[accounts] - max_accounts = 10000000 - file_size_gib = 32 -[log] - path = \"firedancer-dev.log\" - level_stderr = \"INFO\" - level_logfile = \"NOTICE\" - level_flush = \"ERR\" -[rpc] - port = 8123 - extended_tx_metadata_storage = true -" > firedancer-dev.toml - -sudo $FD_DIR/$OBJDIR/bin/firedancer-dev configure init kill --config $(readlink -f firedancer-dev.toml) -sudo $FD_DIR/$OBJDIR/bin/firedancer-dev configure init hugetlbfs --config $(readlink -f firedancer-dev.toml) -sudo $FD_DIR/$OBJDIR/bin/firedancer-dev configure init ethtool-channels --config $(readlink -f firedancer-dev.toml) -sudo $FD_DIR/$OBJDIR/bin/firedancer-dev configure init ethtool-offloads ethtool-loopback --config $(readlink -f firedancer-dev.toml) -sudo $FD_DIR/$OBJDIR/bin/firedancer-dev configure init keys --config $(readlink -f firedancer-dev.toml) - -sudo gdb -iex="set debuginfod enabled on" -ex=r --args $FD_DIR/$OBJDIR/bin/firedancer-dev dev --no-configure --log-path $(readlink -f firedancer-dev.log) --config $(readlink -f firedancer-dev.toml) diff --git a/src/app/fddev/Local.mk b/src/app/fddev/Local.mk index 5442b19a215..ca9aa1ee9cd 100644 --- a/src/app/fddev/Local.mk +++ b/src/app/fddev/Local.mk @@ -36,7 +36,6 @@ monitor: bin $(OBJDIR)/bin/fddev monitor $(MONITOR_ARGS) $(call make-integration-test,test_fddev,tests/test_fddev,fd_fddev fd_fdctl fddev_shared fdctl_shared fdctl_platform fd_discoh fd_disco agave_validator fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util fdctl_version) -$(call run-integration-test,test_fddev) endif endif diff --git a/src/app/firedancer-dev/Local.mk b/src/app/firedancer-dev/Local.mk index 6b7bd18beea..460ff790990 100644 --- a/src/app/firedancer-dev/Local.mk +++ b/src/app/firedancer-dev/Local.mk @@ -28,7 +28,6 @@ endif endif $(call make-integration-test,test_firedancer_dev,tests/test_firedancer_dev,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_vinyl fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version, $(SECP256K1_LIBS) $(ROCKSDB_LIBS) $(OPENSSL_LIBS)) -$(call run-integration-test,test_firedancer_dev) else $(warning firedancer-dev build disabled due to lack of zstd) endif diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 2acb3d2350c..ecb3e83ca96 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -532,11 +532,11 @@ telemetry = true # # The account index is stored in memory and preallocated. A limit # of 1 billion accounts requires approx 100 GB index space. - max_accounts = 30_000_000 + max_accounts = 1_300_000_000 # File size of the account database file (see [paths.accounts]). # The entire account database is physically preallocated. - file_size_gib = 16 + file_size_gib = 800 # The validator keeps account changes that have not yet been # finalized (rooted) by consensus in memory. This setting controls @@ -560,7 +560,7 @@ telemetry = true # # If enabled, allocates [accounts.file_size_gib] memory and ignores # [accounts.{cache_size_gib,max_unrooted_account_size_gib}]. - in_memory_only = true + in_memory_only = false # The provider determines which Linux I/O API is used. # Possible values for the option are: diff --git a/src/app/firedancer/config/mainnet-jito.toml b/src/app/firedancer/config/mainnet-jito.toml index a8876fea752..05d25b0dc10 100644 --- a/src/app/firedancer/config/mainnet-jito.toml +++ b/src/app/firedancer/config/mainnet-jito.toml @@ -8,9 +8,6 @@ ] [consensus] expected_genesis_hash = "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d" -[accounts] - file_size_gib = 600 - max_accounts = 1_100_000_000 [snapshots] [snapshots.sources] servers = [ "http://solana-mainnet-rpc.jumpisolated.com:8899" ] diff --git a/src/app/firedancer/config/mainnet.toml b/src/app/firedancer/config/mainnet.toml index 7c6509b06d6..5ac578c17f7 100644 --- a/src/app/firedancer/config/mainnet.toml +++ b/src/app/firedancer/config/mainnet.toml @@ -8,9 +8,6 @@ ] [consensus] expected_genesis_hash = "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d" -[accounts] - file_size_gib = 600 - max_accounts = 1_100_000_000 [snapshots] [snapshots.sources] servers = [ "http://solana-mainnet-rpc.jumpisolated.com:8899" ] diff --git a/src/flamenco/runtime/tests/run_ledger_backtest.sh b/src/flamenco/runtime/tests/run_ledger_backtest.sh index 733b6cd3b1e..717ea1c8bb9 100755 --- a/src/flamenco/runtime/tests/run_ledger_backtest.sh +++ b/src/flamenco/runtime/tests/run_ledger_backtest.sh @@ -284,6 +284,7 @@ EOF if [[ "$DB" == "funk" ]]; then cat <> ${CONFIG_FILE} [accounts] + in_memory_only = true file_size_gib = $FUNK_PAGES max_accounts = $INDEX_MAX EOF @@ -291,12 +292,10 @@ elif [[ "$DB" == "vinyl" ]]; then if [[ "$INDEX_MAX" -lt "1000000" ]]; then INDEX_MAX=1000000 fi - INDEX_MAX=$(( INDEX_MAX * 2 )) cat <> ${CONFIG_FILE} [accounts] - in_memory_only = false max_accounts = $INDEX_MAX - file_size_gib = $((FUNK_PAGES * 4)) + file_size_gib = $((FUNK_PAGES * 2)) max_unrooted_account_size_gib = 2 cache_size_gib = 10 io_provider = "io_uring"