From 541415e9c3c304917c166e6d504d7a9d9dacf90b Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 5 Jul 2023 00:53:34 -0400 Subject: [PATCH 1/8] WIP: new TOML parser can handle quotes - libautoshell.toml.bash --- src/lib/libautoshell.toml.parser.bash | 54 ++++++++++++++++ test/lib/test_libautoshell.toml.bats | 2 +- test/lib/test_libautoshell.toml.parser.bats | 69 +++++++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/lib/libautoshell.toml.parser.bash create mode 100644 test/lib/test_libautoshell.toml.parser.bats diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash new file mode 100644 index 0000000..74ecb19 --- /dev/null +++ b/src/lib/libautoshell.toml.parser.bash @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +tomlparser.parse() { + local \ + heading="" \ + key="" \ + value="" \ + quote="" \ + mode="read" \ + line char + local -n ref=key + local -n config="${1}" + + _flush() { + [ -n "${key}" ] && \ + config["${heading:+.${heading}}.${key}"]="${value}" + key="" + value="" + } + + while read -r line; do + while read -n1 char; do + case "${char}" in + $'=') + [ -z "${quote-}" ] && \ + local -n ref=value + ;; + "${quote}") + quote="" + ;; + $'"') + quote="${char}" + ;; + $'[') + [ -z "${quote-}" ] && \ + local -n ref=heading + ;; + $']') + [ -z "${quote-}" ] && \ + break + ;; + $'') + [ -z "${quote-}" ] || \ + ref+=" " + ;; + *) + ref+="${char}" + ;; + esac + done <<< "${line}" + local -n ref=key + _flush + done +} \ No newline at end of file diff --git a/test/lib/test_libautoshell.toml.bats b/test/lib/test_libautoshell.toml.bats index d035a4e..7d7b220 100644 --- a/test/lib/test_libautoshell.toml.bats +++ b/test/lib/test_libautoshell.toml.bats @@ -55,4 +55,4 @@ EOC toml.map_value "key1" "heading1" [ "${key1}" = "${expected_value}" ] -} \ No newline at end of file +} diff --git a/test/lib/test_libautoshell.toml.parser.bats b/test/lib/test_libautoshell.toml.parser.bats new file mode 100644 index 0000000..53070a5 --- /dev/null +++ b/test/lib/test_libautoshell.toml.parser.bats @@ -0,0 +1,69 @@ +#!/usr/bin/env bats + +source "src/lib/libautoshell.bash" +include "$(find_lib autoshell.toml.parser)" + +setup() { + declare -Ag TEST_CONFIG + cd "${BATS_TEST_TMPDIR}" +} + +@test "tomlparser.parse: parses key=value pairs from the input file" { + toml_key="key1" + toml_value="value1" + toml_file="./test.toml" + cat <"${toml_file}" +${toml_key}=${toml_value} +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + [ "${TEST_CONFIG[".${toml_key}"]}" = "${toml_value}" ] +} + +@test "tomlparser.parse: parses double quoted string values" { + toml_key="key1" + toml_value="space separated value" + toml_file="./test.toml" + cat <"${toml_file}" +${toml_key}="${toml_value}" +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + echo "${TEST_CONFIG[".${toml_key}"]}" + [ "${TEST_CONFIG[".${toml_key}"]}" = "${toml_value}" ] +} + +@test "tomlparser.parse: parses multiple values from the file" { + toml_key1="key1" + toml_key2="key2" + toml_value1="value1" + toml_value2="value2" + toml_file="./test.toml" + cat <"${toml_file}" +${toml_key1}="${toml_value1}" +${toml_key2}="${toml_value2}" +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + [ "${TEST_CONFIG[".${toml_key1}"]}" = "${toml_value1}" ] + [ "${TEST_CONFIG[".${toml_key2}"]}" = "${toml_value2}" ] +} + +@test "tomlparser.parse: prepends table headers onto key names" { + toml_heading="heading1" + toml_key="key1" + toml_value=v"value1" + toml_file="./test.toml" + cat <"${toml_file}" +[${toml_heading}] +${toml_key}="${toml_value}" +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + echo "${TEST_CONFIG[".${toml_heading}.${toml_key}"]}" + [ "${TEST_CONFIG[".${toml_heading}.${toml_key}"]}" = "${toml_value}" ] +} \ No newline at end of file From 2680619bbc993759d5232b7bed28d0f27cf9d977 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 5 Jul 2023 22:10:36 -0400 Subject: [PATCH 2/8] TOML Parser: Parses array values --- src/lib/libautoshell.toml.parser.bash | 43 +++++++++++++---- test/lib/test_libautoshell.toml.parser.bats | 53 +++++++++++++++++---- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash index 74ecb19..2800b4d 100644 --- a/src/lib/libautoshell.toml.parser.bash +++ b/src/lib/libautoshell.toml.parser.bash @@ -6,18 +6,25 @@ tomlparser.parse() { key="" \ value="" \ quote="" \ + array_index="" \ mode="read" \ line char local -n ref=key local -n config="${1}" - _flush() { - [ -n "${key}" ] && \ - config["${heading:+.${heading}}.${key}"]="${value}" - key="" + _flush_value() { + [ -n "${key}" ] && [ -n "${value}" ] && { + local config_key="${heading:+.${heading}}.${key}${array_index:+[${array_index}]}" + config["${config_key}"]="${value}" + } value="" } + _flush_key() { + _flush_value + key="" + } + while read -r line; do while read -n1 char; do case "${char}" in @@ -32,12 +39,28 @@ tomlparser.parse() { quote="${char}" ;; $'[') - [ -z "${quote-}" ] && \ - local -n ref=heading + [ -z "${quote-}" ] && { + if [[ "$(declare -p ref)" == *"value"* ]]; then + array_index=0 + else + local -n ref=heading + fi + } ;; $']') - [ -z "${quote-}" ] && \ + [ -z "${quote-}" ] && { + log INFO "]" + _flush_key + array_index="" break + } + ;; + $',') + [ -z "${quote-}" ] && { + log INFO "," + _flush_value + array_index=$((array_index + 1)) + } ;; $'') [ -z "${quote-}" ] || \ @@ -48,7 +71,9 @@ tomlparser.parse() { ;; esac done <<< "${line}" - local -n ref=key - _flush + [ -n "${array_index-}" ] || { + local -n ref=key + _flush_key + } done } \ No newline at end of file diff --git a/test/lib/test_libautoshell.toml.parser.bats b/test/lib/test_libautoshell.toml.parser.bats index 53070a5..d9286c8 100644 --- a/test/lib/test_libautoshell.toml.parser.bats +++ b/test/lib/test_libautoshell.toml.parser.bats @@ -18,7 +18,7 @@ TOML tomlparser.parse TEST_CONFIG < "${toml_file}" - [ "${TEST_CONFIG[".${toml_key}"]}" = "${toml_value}" ] + [ "${TEST_CONFIG[.${toml_key}]}" = "${toml_value}" ] } @test "tomlparser.parse: parses double quoted string values" { @@ -31,8 +31,8 @@ TOML tomlparser.parse TEST_CONFIG < "${toml_file}" - echo "${TEST_CONFIG[".${toml_key}"]}" - [ "${TEST_CONFIG[".${toml_key}"]}" = "${toml_value}" ] + echo "${TEST_CONFIG[.${toml_key}]}" + [ "${TEST_CONFIG[.${toml_key}]}" = "${toml_value}" ] } @test "tomlparser.parse: parses multiple values from the file" { @@ -48,14 +48,14 @@ TOML tomlparser.parse TEST_CONFIG < "${toml_file}" - [ "${TEST_CONFIG[".${toml_key1}"]}" = "${toml_value1}" ] - [ "${TEST_CONFIG[".${toml_key2}"]}" = "${toml_value2}" ] + [ "${TEST_CONFIG[.${toml_key1}]}" = "${toml_value1}" ] + [ "${TEST_CONFIG[.${toml_key2}]}" = "${toml_value2}" ] } @test "tomlparser.parse: prepends table headers onto key names" { toml_heading="heading1" toml_key="key1" - toml_value=v"value1" + toml_value="${RANDOM}" toml_file="./test.toml" cat <"${toml_file}" [${toml_heading}] @@ -64,6 +64,43 @@ TOML tomlparser.parse TEST_CONFIG < "${toml_file}" - echo "${TEST_CONFIG[".${toml_heading}.${toml_key}"]}" - [ "${TEST_CONFIG[".${toml_heading}.${toml_key}"]}" = "${toml_value}" ] + echo "${TEST_CONFIG[.${toml_heading}.${toml_key}]}" + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}]}" = "${toml_value}" ] +} + +@test "tomlparser.parse: parses array values into multiple, enumerated CONFIG entries" { + toml_heading="heading1" + toml_key="key1" + toml_value1="${RANDOM}" + toml_value2="${RANDOM}" + toml_file="./test.toml" + cat <"${toml_file}" +[${toml_heading}] +${toml_key} = [ "${toml_value1}", "${toml_value2}" ] +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[0]]}" = "${toml_value1}" ] + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[1]]}" = "${toml_value2}" ] +} + +@test "tomlparser.parse: parses arrays expressed over multiple lines" { + toml_heading="heading1" + toml_key="key1" + toml_value1="${RANDOM}" + toml_value2="${RANDOM}" + toml_file="./test.toml" + cat <"${toml_file}" +[${toml_heading}] +${toml_key} = [ + "${toml_value1}", + "${toml_value2}" +] +TOML + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[0]]}" = "${toml_value1}" ] + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[1]]}" = "${toml_value2}" ] } \ No newline at end of file From 7f614b25142897b60f893970b130e8122009f63c Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Wed, 5 Jul 2023 23:17:59 -0400 Subject: [PATCH 3/8] WIP: Refator libautoshell.toml to use new char based toml parser --- src/lib/libautoshell.toml.bash | 52 +++++++++------------------ src/lib/libautoshell.toml.parser.bash | 6 ++-- src/lib/libautotask.config.bash | 24 ++++++------- test/lib/test_libautoshell.toml.bats | 12 +++---- test/lib/test_libautotask.config.bats | 13 ++++--- 5 files changed, 44 insertions(+), 63 deletions(-) diff --git a/src/lib/libautoshell.toml.bash b/src/lib/libautoshell.toml.bash index 2c56c40..0197d05 100644 --- a/src/lib/libautoshell.toml.bash +++ b/src/lib/libautoshell.toml.bash @@ -1,55 +1,37 @@ #!/usr/bin/env bash -export TOML_PREFIX="TOML" +export TOML_CONFIG_VAR="TOML_CONFIG" toml.load() { # toml_file[, toml_prefix=$TOML_PREFIX] - local \ - current_heading="default" \ - current_key \ - current_value \ - global_var_name \ - line \ toml_file="${1}" \ - toml_prefix="${2:-"${TOML_PREFIX}"}" - - while read -r line; do - case "${line}" in - \[*\]) - current_heading="$(sed -e 's/\[\(.*\)\]/\1/' <<< "${line}")" - ;; - *\ \=\ *) - current_key="$(sed -e 's/^\(.*\) = .*$/\1/' <<< "${line}")" - current_value="$(sed -e 's/^.* = \(.*\)$/\1/' <<< "${line}")" - - if [ -n "${current_value}" ]; then - global_var_name="${toml_prefix}_${current_heading/\./_}_KEY_${current_key}" - declare -gx "${global_var_name}" - local -n ref_var="${global_var_name}" - ref_var="${current_value}" - fi - ;; - esac - done < <(cat "${toml_file}"; echo) + config_var="${2:-"${TOML_CONFIG_VAR}"}" + + include "$(find_lib autoshell.toml.parser)" + + declare -gA "${config_var}" + + tomlparser.parse "${config_var}" < "${toml_file}" } export -f toml.load toml.get_value() { # toml_key, toml_section[, toml_prefix=$TOML_PREFIX] local \ toml_key="${1}" \ - toml_section="${2}" \ - toml_prefix="${3:-"${TOML_PREFIX}"}" + config_var_name="${2:-"${TOML_CONFIG_VAR}"}" - toml.map_value "${toml_key}" "${toml_section}" "${toml_prefix}" TOML_VALUE + toml.map_value "${toml_key}" TOML_VALUE "${config_var_name}" echo "${TOML_VALUE-}" } -toml.map_value() { # toml_key, toml_section[, toml_prefix=$TOML_PREFIX, dest_var=$toml_key] +toml.map_value() { # toml_key, dest_var[, config_var=$TOML_CONFIG_VAR] local \ toml_key="${1}" \ - toml_section="${2}" \ - toml_prefix="${3:-"${TOML_PREFIX}"}" \ - dest_var="${4:-${1}}" + dest_var="${2}" \ + config_var_name="${3:-"${TOML_CONFIG_VAR}"}" \ + config_value + local -n config_var="${config_var_name}" + config_value="${config_var["${toml_key}"]-}" - declare -gn "${dest_var}=${toml_prefix}_${toml_section}_KEY_${toml_key}" + declare -g "${dest_var}=${config_value}" } export -f toml.map_value diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash index 2800b4d..11fdfad 100644 --- a/src/lib/libautoshell.toml.parser.bash +++ b/src/lib/libautoshell.toml.parser.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -tomlparser.parse() { +tomlparser.parse() { # config_var <<< TOML Data local \ heading="" \ key="" \ @@ -49,7 +49,6 @@ tomlparser.parse() { ;; $']') [ -z "${quote-}" ] && { - log INFO "]" _flush_key array_index="" break @@ -57,12 +56,11 @@ tomlparser.parse() { ;; $',') [ -z "${quote-}" ] && { - log INFO "," _flush_value array_index=$((array_index + 1)) } ;; - $'') + $''|$' ') [ -z "${quote-}" ] || \ ref+=" " ;; diff --git a/src/lib/libautotask.config.bash b/src/lib/libautotask.config.bash index f0afa97..eda5746 100644 --- a/src/lib/libautotask.config.bash +++ b/src/lib/libautotask.config.bash @@ -1,8 +1,8 @@ #!/usr/bin/env bash export \ - TASK_CONFIG_PREFIX="TASK_CONFIG" \ - TASK_USER_CONFIG_PREFIX="TASK_USER_CONFIG" + TASK_CONFIG_VAR="TASK_CONFIG" \ + TASK_USER_CONFIG_VAR="TASK_USER_CONFIG" task.load_config() { local \ task_name="${1}" \ @@ -12,12 +12,12 @@ task.load_config() { IFS="." [ "${task_name}" = "${TASK_MAIN-}" ] && [ -f "./project.toml" ] && \ - toml.load "./project.toml" "${TASK_USER_CONFIG_PREFIX}" + toml.load "./project.toml" "${TASK_USER_CONFIG_VAR}" for task_part in ${task_name}; do task="${task:+${task}.}${task_part}" if toml_file="$(task.find_file "${task}" toml)"; then - toml.load "${toml_file}" "${TASK_CONFIG_PREFIX}" + toml.load "${toml_file}" "${TASK_CONFIG_VAR}" fi done } @@ -30,23 +30,21 @@ task.get_config() { # key_name[, task_name=$TASK_NAME] export "${key_name}" local -n key_ref="${key_name}" - task.get_config^try_prefix "${TASK_USER_CONFIG_PREFIX}" + task.get_config^try_prefix "${TASK_USER_CONFIG_VAR}" [ -n "${key_ref-}" ] || \ - task.get_config^try_prefix "${TASK_CONFIG_PREFIX}" + task.get_config^try_prefix "${TASK_CONFIG_VAR}" } task.get_config^try_prefix() { local \ - toml_prefix="${1}" \ - toml_section="" \ + config_var="${1}" \ + toml_key="" \ task_part \ IFS="." - local -a toml_config=() for task_part in ${task_name}; do - toml_section="${toml_section:+${toml_section}_}${task_part}" - toml_config=("${key_name}" "${toml_section}" "${toml_prefix}") - if [ -n "$(toml.get_value "${toml_config[@]}")" ]; then - toml.map_value "${toml_config[@]}" + toml_key="${toml_key-}.${task_part}" + if [ -n "$(toml.get_value "${toml_key}.${key_name}" "${config_var}")" ]; then + toml.map_value "${toml_key}.${key_name}" "${key_name}" "${config_var}" fi done } diff --git a/test/lib/test_libautoshell.toml.bats b/test/lib/test_libautoshell.toml.bats index 7d7b220..bc1f07c 100644 --- a/test/lib/test_libautoshell.toml.bats +++ b/test/lib/test_libautoshell.toml.bats @@ -13,10 +13,10 @@ EOC toml.load "${toml_file}" - [ "${TOML_heading1_KEY_key1}" = "${expected_value}" ] + [ "${TOML_CONFIG[.heading1.key1]}" = "${expected_value}" ] } -@test "toml.load: can override variable prefix values are loaded into" { +@test "toml.load: can override config variable values are loaded into" { toml_file="${BATS_TEST_TMPDIR}/config.toml" expected_value="${RANDOM}" cat <"${toml_file}" @@ -24,9 +24,9 @@ EOC key1 = ${expected_value} EOC - toml.load "${toml_file}" "MY_PREFIX" + toml.load "${toml_file}" "MY_CONFIG" - [ "${MY_PREFIX_heading1_KEY_key1}" = "${expected_value}" ] + [ "${MY_CONFIG[.heading1.key1]}" = "${expected_value}" ] } @test "toml.get_value: echos loaded toml environment variable value for the header and key" { @@ -39,7 +39,7 @@ EOC toml.load "${toml_file}" - [ "$(toml.get_value "key1" "heading1")" = "${expected_value}" ] + [ "$(toml.get_value ".heading1.key1")" = "${expected_value}" ] } @test "toml.map_value: references loaded toml environment variable to variable named toml_key" { @@ -52,7 +52,7 @@ EOC toml.load "${toml_file}" - toml.map_value "key1" "heading1" + toml.map_value ".heading1.key1" "key1" [ "${key1}" = "${expected_value}" ] } diff --git a/test/lib/test_libautotask.config.bats b/test/lib/test_libautotask.config.bats index e8887ca..85c3b41 100644 --- a/test/lib/test_libautotask.config.bats +++ b/test/lib/test_libautotask.config.bats @@ -25,6 +25,7 @@ build_config() { task_name="task1" task_config_key="key1" expected_value="${RANDOM}" + loaded_config_key=".${task_name}.${task_config_key}" cat <"$(build_config "${task_name}")" [${task_name}] @@ -33,7 +34,7 @@ EOC task.load_config "${task_name}" - [ "$(toml.get_value "${task_config_key}" "${task_name}" "${TASK_CONFIG_PREFIX}")" = "${expected_value}" ] + [ "$(toml.get_value "${loaded_config_key}" "${TASK_CONFIG_VAR}")" = "${expected_value}" ] } @test "task.load_config: also loads parent config when present" { @@ -41,6 +42,7 @@ EOC task_name="${parent_task_name}.task1" task_config_key="key1" expected_value="${RANDOM}" + loaded_config_key=".${parent_task_name}.${task_config_key}" cat <"$(build_config "${parent_task_name}")" [${parent_task_name}] @@ -49,14 +51,15 @@ EOC task.load_config "${task_name}" - [ "$(toml.get_value "${task_config_key}" "${parent_task_name}" "${TASK_CONFIG_PREFIX}")" = "${expected_value}" ] + [ "$(toml.get_value "${loaded_config_key}" "${TASK_CONFIG_VAR}")" = "${expected_value}" ] } -@test "task.load_config: loads user config into TASK_USER_CONFIG_PREFIX" { +@test "task.load_config: loads user config into TASK_USER_CONFIG_VAR" { task_name="task1" TASK_MAIN="${task_name}" task_config_key="key1" expected_value="${RANDOM}" + loaded_config_key=".${task_name}.${task_config_key}" cat <"./project.toml" [${task_name}] @@ -65,7 +68,7 @@ EOC task.load_config "${task_name}" - [ "$(toml.get_value "${task_config_key}" "${task_name}" "${TASK_USER_CONFIG_PREFIX}")" = "${expected_value}" ] + [ "$(toml.get_value "${loaded_config_key}" "${TASK_USER_CONFIG_VAR}")" = "${expected_value}" ] } @test "task.load_config: does not load user config when not running as main task" { @@ -80,7 +83,7 @@ EOC task.load_config "${task_name}" - [ -z "$(toml.get_value "${task_config_key}" "${task_name}" "${TASK_USER_CONFIG_PREFIX}")" ] + [ -z "$(toml.get_value "${task_config_key}" "${task_name}" "${TASK_USER_CONFIG_VAR}")" ] } @test "task.get_config: copies the TOML loaded config value into the key name variable" { From 1749503b924d8ec8d154e185fbe03fcd2815f984 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Thu, 6 Jul 2023 19:18:42 -0400 Subject: [PATCH 4/8] Fix remaining failing tests from refactor --- src/lib/libautoshell.toml.bash | 5 +++-- test/lib/test_libautoshell.toml.bats | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/libautoshell.toml.bash b/src/lib/libautoshell.toml.bash index 0197d05..d07a232 100644 --- a/src/lib/libautoshell.toml.bash +++ b/src/lib/libautoshell.toml.bash @@ -30,8 +30,9 @@ toml.map_value() { # toml_key, dest_var[, config_var=$TOML_CONFIG_VAR] config_var_name="${3:-"${TOML_CONFIG_VAR}"}" \ config_value local -n config_var="${config_var_name}" - config_value="${config_var["${toml_key}"]-}" + [ -n "${config_var[*]-}" ] && \ + config_value="${config_var["${toml_key}"]-}" - declare -g "${dest_var}=${config_value}" + declare -g "${dest_var}=${config_value-}" } export -f toml.map_value diff --git a/test/lib/test_libautoshell.toml.bats b/test/lib/test_libautoshell.toml.bats index bc1f07c..df92593 100644 --- a/test/lib/test_libautoshell.toml.bats +++ b/test/lib/test_libautoshell.toml.bats @@ -56,3 +56,11 @@ EOC [ "${key1}" = "${expected_value}" ] } + +@test "toml.map_value: gracefully handles the config variable being unset, clearing stored values" { + my_var="stale" + + toml.map_value ".doop" "my_var" "UNDEFINED_CONFIG_VAR" + + [ -z "${my_var-}"] +} \ No newline at end of file From c945af97022a4af11cbd59aadb240b20eddc06dd Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Fri, 7 Jul 2023 19:28:56 -0400 Subject: [PATCH 5/8] toml.map_value can map array values - bumped to verion 0.1.0 as first minimum usable code set --- VERSION | 2 +- auto => project | 0 src/lib/libautoshell.toml.bash | 29 +++++++++++++++++++--------- test/lib/test_libautoshell.toml.bats | 26 ++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 11 deletions(-) rename auto => project (100%) diff --git a/VERSION b/VERSION index 3eefcb9..6e8bf73 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +0.1.0 diff --git a/auto b/project similarity index 100% rename from auto rename to project diff --git a/src/lib/libautoshell.toml.bash b/src/lib/libautoshell.toml.bash index d07a232..a7dc206 100644 --- a/src/lib/libautoshell.toml.bash +++ b/src/lib/libautoshell.toml.bash @@ -26,13 +26,24 @@ toml.get_value() { # toml_key, toml_section[, toml_prefix=$TOML_PREFIX] toml.map_value() { # toml_key, dest_var[, config_var=$TOML_CONFIG_VAR] local \ toml_key="${1}" \ - dest_var="${2}" \ - config_var_name="${3:-"${TOML_CONFIG_VAR}"}" \ - config_value - local -n config_var="${config_var_name}" - [ -n "${config_var[*]-}" ] && \ - config_value="${config_var["${toml_key}"]-}" - - declare -g "${dest_var}=${config_value-}" + dest_var_name="${2}" \ + config_var_name="${3:-"${TOML_CONFIG_VAR}"}" + local -n config_ref="${config_var_name}" + local -n var_ref="${dest_var_name}" + declare -g "${dest_var_name}" + var_ref="" + + [ -n "${config_ref[*]-}" ] || \ + return 0 + + var_ref="${config_ref["${toml_key}"]-}" + + [ -n "${var_ref-}" ] || { + [ "$(type -t map.keys)" = "function" ] || \ + include "$(find_lib map)" + declare -ga "${dest_var_name}=()" + while read -r key; do var_ref+=("${config_ref[${key}]}"); done < \ + <( map.keys config_ref | grep "${toml_key}\[[0-9]*\]" | sort ) + } } -export -f toml.map_value +export -f toml.map_value \ No newline at end of file diff --git a/test/lib/test_libautoshell.toml.bats b/test/lib/test_libautoshell.toml.bats index df92593..4d323ca 100644 --- a/test/lib/test_libautoshell.toml.bats +++ b/test/lib/test_libautoshell.toml.bats @@ -62,5 +62,29 @@ EOC toml.map_value ".doop" "my_var" "UNDEFINED_CONFIG_VAR" - [ -z "${my_var-}"] + log INFO "my_var: ${my_var}" + + [ -z "${my_var-}" ] +} + +@test "toml.map_value: maps array keys to array variables" { + toml_file="${BATS_TEST_TMPDIR}/config.toml" + expected_value1="${RANDOM}" + expected_value2="${RANDOM}" + cat <"${toml_file}" +[heading1] +key1 = [ ${expected_value1}, ${expected_value2} ] +EOC + + toml.load "${toml_file}" + + toml.map_value ".heading1.key1" "key1" + + echo "expected: [ ${expected_value1} ${expected_value2} ]" + echo "key1[${#key1[@]}]: [ ${key1[*]} ]" + + [ "${#key1[@]}" -eq 2 ] + + [ "${key1[0]}" = "${expected_value1}" ] + [ "${key1[1]}" = "${expected_value2}" ] } \ No newline at end of file From 26dab5255eb2cf3d3d33cb78c94ee5301354f2d9 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Fri, 7 Jul 2023 23:49:38 -0400 Subject: [PATCH 6/8] Refactor toml parser to be able to catch errors --- src/lib/libautoshell.exec.bash | 16 +- src/lib/libautoshell.toml.parser.bash | 263 +++++++++++++++----- src/lib/libmap.bash | 10 + test/bin/test_autotask.bats | 11 +- test/lib/test_libautoshell.toml.bats | 10 +- test/lib/test_libautoshell.toml.parser.bats | 45 +++- test/lib/test_libautotask.config.bats | 16 +- 7 files changed, 269 insertions(+), 102 deletions(-) create mode 100644 src/lib/libmap.bash diff --git a/src/lib/libautoshell.exec.bash b/src/lib/libautoshell.exec.bash index 12bf787..8bd520f 100644 --- a/src/lib/libautoshell.exec.bash +++ b/src/lib/libautoshell.exec.bash @@ -28,13 +28,15 @@ export -f try # 1 and exits the current process ############################### fatal() { - local _message="${1}" - - if type log &>/dev/null; then - log FATAL "${_message}" "${FUNCNAME[1]-}" - else - echo "FATAL: ${_message}" - fi + local _message="${1-}" + + [ -n "${_message}" ] && { + if type log &>/dev/null; then + log FATAL "${_message}" "${FUNCNAME[1]-}" + else + echo "FATAL: ${_message}" + fi + } exit 1 } diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash index 11fdfad..349acc4 100644 --- a/src/lib/libautoshell.toml.parser.bash +++ b/src/lib/libautoshell.toml.parser.bash @@ -1,77 +1,212 @@ #!/usr/bin/env bash tomlparser.parse() { # config_var <<< TOML Data - local \ + declare -g \ heading="" \ key="" \ value="" \ quote="" \ array_index="" \ - mode="read" \ + line_index="1" \ + mode="key" \ line char - local -n ref=key - local -n config="${1}" - - _flush_value() { - [ -n "${key}" ] && [ -n "${value}" ] && { - local config_key="${heading:+.${heading}}.${key}${array_index:+[${array_index}]}" - config["${config_key}"]="${value}" - } - value="" - } - - _flush_key() { - _flush_value - key="" - } + declare -gn ref=key + declare -gn config="${1}" while read -r line; do - while read -n1 char; do - case "${char}" in - $'=') - [ -z "${quote-}" ] && \ - local -n ref=value - ;; - "${quote}") - quote="" - ;; - $'"') - quote="${char}" - ;; - $'[') - [ -z "${quote-}" ] && { - if [[ "$(declare -p ref)" == *"value"* ]]; then - array_index=0 - else - local -n ref=heading - fi - } - ;; - $']') - [ -z "${quote-}" ] && { - _flush_key - array_index="" - break - } - ;; - $',') - [ -z "${quote-}" ] && { - _flush_value - array_index=$((array_index + 1)) - } - ;; - $''|$' ') - [ -z "${quote-}" ] || \ - ref+=" " - ;; - *) - ref+="${char}" - ;; - esac - done <<< "${line}" - [ -n "${array_index-}" ] || { - local -n ref=key - _flush_key - } + tomlparser.parse_line "${line}" + line_index=$(( line_index + 1 )) done +} + +tomlparser.parse_line() { + local \ + line="${1}" \ + last_char="" \ + char_index=0 + + while read -n1 char; do + tomlparser.parse_char "${char_index}" "${char-}" "${last_char-}" + char_index=$(( char_index + 1 )) + last_char="${char-}" + done <<< "${line}" + case "${mode}" in + array) + ;; + *) + tomlparser.flush_key + mode=key + ;; + esac +} + +tomlparser.parse_char() { + local \ + char_index="${1}" \ + char="${2:- }" \ + last_char="${3:- }" + + case "${mode}" in + key) + if [ -n "${quote-}" ]; then + tomlparser.parse_char^quote key "${char}" + else + tomlparser.parse_char^key "${char}" + fi + ;; + heading) + if [ -n "${quote-}" ]; then + tomlparser.parse_char^quote heading "${char}" + else + tomlparser.parse_char^heading "${char}" + fi + ;; + value) + if [ -n "${quote-}" ]; then + tomlparser.parse_char^quote value "${char}" + else + tomlparser.parse_char^value "${char}" + fi + ;; + array) + [ -n "${array_index-}" ] || array_index=0 + if [ -n "${quote-}" ]; then + tomlparser.parse_char^quote value "${char}" + else + tomlparser.parse_char^array "${char}" + fi + ;; + line_return) + tomlparser.parse_char^line_return "${char}" + ;; + comment) + ;; + esac +} + +tomlparser.parse_char^quote() { + local -n ref="${1}" + local char="${2}" + + case "${char}" in + "${quote}") + quote="" + ;; + $''|$' ') + ref+=" " + ;; + *) + ref+="${char}" + ;; + esac +} + +tomlparser.parse_char^key() { + case "${1}" in + $'[') + heading="" + mode=heading + ;; + $'=') + mode=value + ;; + $'"') + quote="${char}" + ;; + $' ') + ;; + *) + [ "${last_char}" = " " ] && [ "${char_index}" -gt 0 ] && \ + tomlparser.parse_error "Unquoted keys cannot have spaces" + key+="${char}" + ;; + esac +} + +tomlparser.parse_char^heading() { + case "${1}" in + $']') + mode=line_return + ;; + $'"') + quote="${char}" + ;; + $' ') + ;; + *) + heading+="${char}" + ;; + esac +} + +tomlparser.parse_char^value() { + case "${1}" in + $'[') + mode=array + ;; + $'"') + quote="${char}" + ;; + $' ') + ;; + $'#') + mode=comment + ;; + *) + tomlparser.parse_error "Unquoted string" + ;; + esac +} + +tomlparser.parse_char^array() { + case "${1}" in + $']') + tomlparser.flush_key + mode=line_return + ;; + $',') + tomlparser.flush_value + array_index=$(( array_index + 1 )) + ;; + $'"') + quote="${char}" + ;; + $' ') + ;; + *) + tomlparser.parse_error "Unquoted string" + ;; + esac +} + +tomlparser.parse_char^line_return() { + case "${1}" in + $' ') + ;; + $'#') + mode[-1]="comment" + ;; + *) + tomlparser.parse_error "Unexpected symbol: ${char}" + ;; + esac +} + +tomlparser.flush_value() { + [ -n "${key}" ] && [ -n "${value}" ] && { + local config_key="${heading:+.${heading}}.${key}${array_index:+[${array_index}]}" + local config_value="$(envsubst <<< "${value}")" + config["${config_key}"]="${config_value}" + } + value="" +} + +tomlparser.flush_key() { + tomlparser.flush_value + key="" +} + +tomlparser.parse_error() { + log FATAL "${1}" "tomlparser: line ${line_index}, char ${char_index}" + fatal } \ No newline at end of file diff --git a/src/lib/libmap.bash b/src/lib/libmap.bash new file mode 100644 index 0000000..7c1413a --- /dev/null +++ b/src/lib/libmap.bash @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +map.keys() { + local -n array="${1}" + local key + for key in "${!array[@]}"; do + echo "${key}" + done +} +export -f map.keys \ No newline at end of file diff --git a/test/bin/test_autotask.bats b/test/bin/test_autotask.bats index 0258a0e..7dc57cf 100644 --- a/test/bin/test_autotask.bats +++ b/test/bin/test_autotask.bats @@ -332,7 +332,7 @@ EOT cat <"$(build_task.config "${task_name}")" [task1] -test_value = ${expected_task_output} +test_value = "${expected_task_output}" EOC run "${AUTOTASK}" "${task_name}" @@ -358,18 +358,19 @@ EOT cat <"$(build_task.config "${task_name}")" [task1] -test_value = default_value +test_value = "default_value" EOC cat < "./project.toml" [task1] -test_value = ${expected_task_output} +test_value = "${expected_task_output}" EOC - run -0 "${AUTOTASK}" "${task_name}" + run "${AUTOTASK}" "${task_name}" echo "${output}" + [ "${status}" -eq 0 ] [ "${output}" = "${expected_task_output}" ] } @@ -390,7 +391,7 @@ EOT cat <"$(build_task.config "${parent_task_name}")" [task1] -test_value = ${expected_task_output} +test_value = "${expected_task_output}" EOC run -0 "${AUTOTASK}" "${child_task_name}" diff --git a/test/lib/test_libautoshell.toml.bats b/test/lib/test_libautoshell.toml.bats index 4d323ca..80f4dac 100644 --- a/test/lib/test_libautoshell.toml.bats +++ b/test/lib/test_libautoshell.toml.bats @@ -8,7 +8,7 @@ source "src/lib/libautoshell.toml.bash" expected_value="${RANDOM}" cat <"${toml_file}" [heading1] -key1 = ${expected_value} +key1 = "${expected_value}" EOC toml.load "${toml_file}" @@ -21,7 +21,7 @@ EOC expected_value="${RANDOM}" cat <"${toml_file}" [heading1] -key1 = ${expected_value} +key1 = "${expected_value}" EOC toml.load "${toml_file}" "MY_CONFIG" @@ -34,7 +34,7 @@ EOC expected_value="${RANDOM}" cat <"${toml_file}" [heading1] -key1 = ${expected_value} +key1 = "${expected_value}" EOC toml.load "${toml_file}" @@ -47,7 +47,7 @@ EOC expected_value="${RANDOM}" cat <"${toml_file}" [heading1] -key1 = ${expected_value} +key1 = "${expected_value}" EOC toml.load "${toml_file}" @@ -73,7 +73,7 @@ EOC expected_value2="${RANDOM}" cat <"${toml_file}" [heading1] -key1 = [ ${expected_value1}, ${expected_value2} ] +key1 = [ "${expected_value1}", "${expected_value2}" ] EOC toml.load "${toml_file}" diff --git a/test/lib/test_libautoshell.toml.parser.bats b/test/lib/test_libautoshell.toml.parser.bats index d9286c8..6f93748 100644 --- a/test/lib/test_libautoshell.toml.parser.bats +++ b/test/lib/test_libautoshell.toml.parser.bats @@ -1,5 +1,7 @@ #!/usr/bin/env bats +bats_require_minimum_version "1.5.0" + source "src/lib/libautoshell.bash" include "$(find_lib autoshell.toml.parser)" @@ -8,19 +10,6 @@ setup() { cd "${BATS_TEST_TMPDIR}" } -@test "tomlparser.parse: parses key=value pairs from the input file" { - toml_key="key1" - toml_value="value1" - toml_file="./test.toml" - cat <"${toml_file}" -${toml_key}=${toml_value} -TOML - - tomlparser.parse TEST_CONFIG < "${toml_file}" - - [ "${TEST_CONFIG[.${toml_key}]}" = "${toml_value}" ] -} - @test "tomlparser.parse: parses double quoted string values" { toml_key="key1" toml_value="space separated value" @@ -103,4 +92,34 @@ TOML [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[0]]}" = "${toml_value1}" ] [ "${TEST_CONFIG[.${toml_heading}.${toml_key}[1]]}" = "${toml_value2}" ] +} + +@test "tomlparser.parse: substitutes valid variable expressions with their values" { + toml_heading="heading1" + toml_key="key1" + expected_value="${RANDOM}" + toml_file="./test.toml" + cat <"${toml_file}" +[${toml_heading}] +${toml_key}="\${ENV_VAR}" +TOML + export ENV_VAR="${expected_value}" + + tomlparser.parse TEST_CONFIG < "${toml_file}" + + echo "${TEST_CONFIG[.${toml_heading}.${toml_key}]}" + [ "${TEST_CONFIG[.${toml_heading}.${toml_key}]}" = "${expected_value}" ] +} + +@test "tomlparser.parse: treats unquoted values as an error" { + toml_file="./test.toml" + cat <"${toml_file}" +key1=value1 +TOML + + run ! tomlparser.parse TEST_CONFIG < "${toml_file}" + + echo "output: ${output}" + + [ "${output}" = "[FATAL] tomlparser: line 1, char 5: Unquoted string" ] } \ No newline at end of file diff --git a/test/lib/test_libautotask.config.bats b/test/lib/test_libautotask.config.bats index 85c3b41..373312a 100644 --- a/test/lib/test_libautotask.config.bats +++ b/test/lib/test_libautotask.config.bats @@ -29,7 +29,7 @@ build_config() { cat <"$(build_config "${task_name}")" [${task_name}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${task_name}" @@ -46,7 +46,7 @@ EOC cat <"$(build_config "${parent_task_name}")" [${parent_task_name}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${task_name}" @@ -63,7 +63,7 @@ EOC cat <"./project.toml" [${task_name}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${task_name}" @@ -93,7 +93,7 @@ EOC cat <"$(build_config "${task_name}")" [${task_name}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${task_name}" @@ -112,7 +112,7 @@ EOC cat <"$(build_config "${TASK_NAME}")" [${TASK_NAME}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${TASK_NAME}" @@ -132,7 +132,7 @@ EOC cat <"$(build_config "${parent_task}")" [${parent_task}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${TASK_NAME}" @@ -153,11 +153,11 @@ EOC cat <"$(build_config "${TASK_NAME}")" [${TASK_NAME}] -${task_config_key} = default +${task_config_key} = "default" EOC cat <"./project.toml" [${task_parent_name}] -${task_config_key} = ${expected_value} +${task_config_key} = "${expected_value}" EOC task.load_config "${TASK_NAME}" From b874eb090b4824ea983424b45b1bf07c2f6ee2f4 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Sat, 8 Jul 2023 01:04:44 -0400 Subject: [PATCH 7/8] Initial working package task --- project.toml | 16 +++++++------- src/bin/autotask | 2 +- src/lib/libautoshell.toml.parser.bash | 7 ++++++- tasks/hello_world/hello_world.toml | 2 +- tasks/package/package.bash | 16 ++++++++++++++ tasks/package/package.toml | 8 +++++++ tasks/package/stage.bash | 30 +++++++++++++++++++++++++++ 7 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 tasks/package/package.bash create mode 100644 tasks/package/package.toml create mode 100644 tasks/package/stage.bash diff --git a/project.toml b/project.toml index 26024d5..0df7080 100644 --- a/project.toml +++ b/project.toml @@ -1,13 +1,13 @@ [project] -name = structured-shell -maintainer = Kyle Smith -description = ''' -Structured Shell (STSH) is a script execution framework extending the power of - BASH Shell scripting with safer programming paradigms. -''' +name = "structured-shell" +maintainer = "Kyle Smith " +# description = ''' +# Structured Shell (STSH) is a script execution framework extending the power of +# BASH Shell scripting with safer programming paradigms. +# ''' [build] -build_root = ${PROJECT_ROOT}/build +build_root = "${PROJECT_ROOT}/build" [hello_world] -greeting = Salutations +greeting = "Salutations" diff --git a/src/bin/autotask b/src/bin/autotask index 9234256..461d348 100755 --- a/src/bin/autotask +++ b/src/bin/autotask @@ -18,7 +18,7 @@ __script_parse_opts() { declare -g TASK_FILE - LOG_CONTEXT_BUILDER="autotask:${TASK_NAME}:" + # LOG_CONTEXT_BUILDER="autotask:${TASK_NAME}:" [ -n "${TASK_MAIN-}" ] || \ TASK_MAIN="${TASK_NAME}" } diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash index 349acc4..72adacb 100644 --- a/src/lib/libautoshell.toml.parser.bash +++ b/src/lib/libautoshell.toml.parser.bash @@ -25,7 +25,7 @@ tomlparser.parse_line() { last_char="" \ char_index=0 - while read -n1 char; do + while read -rN1 char; do tomlparser.parse_char "${char_index}" "${char-}" "${last_char-}" char_index=$(( char_index + 1 )) last_char="${char-}" @@ -46,6 +46,8 @@ tomlparser.parse_char() { char="${2:- }" \ last_char="${3:- }" + [ "${char}" = $'\n' ] && return 0 + case "${mode}" in key) if [ -n "${quote-}" ]; then @@ -113,6 +115,9 @@ tomlparser.parse_char^key() { $'"') quote="${char}" ;; + $'#') + mode=comment + ;; $' ') ;; *) diff --git a/tasks/hello_world/hello_world.toml b/tasks/hello_world/hello_world.toml index 515a3ee..4875e63 100644 --- a/tasks/hello_world/hello_world.toml +++ b/tasks/hello_world/hello_world.toml @@ -1,2 +1,2 @@ [hello_world] -greeting = Hello +greeting = "Hello" diff --git a/tasks/package/package.bash b/tasks/package/package.bash new file mode 100644 index 0000000..f4c8ee1 --- /dev/null +++ b/tasks/package/package.bash @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +task.dependencies() { + depends_on "package.stage" +} + +task.exec() { + local archive dir + + task.get_config archive + task.get_config dir package.stage + + log INFO "Creating archive: ${archive}" + mkdir -p "$(dirname "${archive}")" + tar -czf "${archive}" -C "${dir}" ./ +} \ No newline at end of file diff --git a/tasks/package/package.toml b/tasks/package/package.toml new file mode 100644 index 0000000..ca63917 --- /dev/null +++ b/tasks/package/package.toml @@ -0,0 +1,8 @@ +[package] +archive = "${PROJECT_ROOT}/build/autoshell.tar.gz" +stage.dir = "${PROJECT_ROOT}/build/stage" +stage.files = [ + "${PROJECT_ROOT}/src/lib:lib/autoshell", + "${PROJECT_ROOT}/src/bin:bin", + "${PROJECT_ROOT}/tasks:share/autoshell/tasks" +] diff --git a/tasks/package/stage.bash b/tasks/package/stage.bash new file mode 100644 index 0000000..043cabd --- /dev/null +++ b/tasks/package/stage.bash @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +task.exec() { + local dir + local -a files + + task.get_config dir + task.get_config files + + log INFO "Creating staging directory: ${dir}" + rm -rf "${dir}" + mkdir -p "${dir}" + + log INFO "Copying files:" + cd "${dir}" || \ + fatal "Could not enter staging directory" + for cp_expr in "${files[@]}"; do + [ -n "${cp_expr}" ] || \ + continue + + log INFO " - ${cp_expr}" + local IFS=":" + local -a cp_files=(${cp_expr}) + [ "${#cp_files[@]}" -gt 1 ] || \ + fatal "stage.files: ${cp_expr}: destination not specified" + + mkdir -p "$(dirname "${cp_files[-1]}")" + cp -r "${cp_files[@]}" + done +} \ No newline at end of file From 6d211d6a01977e2b53250d5ba9fe472e09bfa907 Mon Sep 17 00:00:00 2001 From: Kyle Smith Date: Fri, 21 Jul 2023 19:29:45 -0400 Subject: [PATCH 8/8] Expand unexpected token error --- src/lib/libautoshell.toml.parser.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/libautoshell.toml.parser.bash b/src/lib/libautoshell.toml.parser.bash index 72adacb..0bf3ed8 100644 --- a/src/lib/libautoshell.toml.parser.bash +++ b/src/lib/libautoshell.toml.parser.bash @@ -122,7 +122,7 @@ tomlparser.parse_char^key() { ;; *) [ "${last_char}" = " " ] && [ "${char_index}" -gt 0 ] && \ - tomlparser.parse_error "Unquoted keys cannot have spaces" + tomlparser.parse_error "Unexpected char: ${char}: Unquoted keys cannot have spaces" key+="${char}" ;; esac