diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7ff91d2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + rust-version: [stable, beta] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust-version }} + components: rustfmt, clippy + + - name: Cache Rust dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Run clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Run unit tests + run: cargo test --lib + + - name: Build binaries + run: cargo build --release + + - name: Run integration tests + run: cargo test --test integration_test + + build: + runs-on: ubuntu-latest + needs: test + + strategy: + matrix: + target: + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-gnu + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + cargo install cross + + - name: Build for ${{ matrix.target }} + run: | + if [ "${{ matrix.target }}" = "x86_64-unknown-linux-gnu" ]; then + cargo build --release --target ${{ matrix.target }} + else + cross build --release --target ${{ matrix.target }} + fi + + - name: Upload build artifacts + uses: actions/upload-artifact@v3 + with: + name: binaries-${{ matrix.target }} + path: target/${{ matrix.target }}/release/flare-* + + docker: + runs-on: ubuntu-latest + needs: test + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + run: | + docker build -t flare-tools:$(git describe --tags --dirty --always) . + + - name: Test Docker image + run: | + docker run --rm flare-tools:$(git describe --tags --dirty --always) flare-admin --help + docker run --rm flare-tools:$(git describe --tags --dirty --always) flare-stats --help \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..748e073 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,92 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + + strategy: + matrix: + target: + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-gnu + - x86_64-apple-darwin + - aarch64-apple-darwin + + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Get tag name + id: tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Install cross-compilation tools + run: | + cargo install cross + + - name: Build for ${{ matrix.target }} + run: | + if [[ "${{ matrix.target }}" == *"darwin"* ]]; then + # For macOS targets, we need special handling + cross build --release --target ${{ matrix.target }} + elif [ "${{ matrix.target }}" = "x86_64-unknown-linux-gnu" ]; then + cargo build --release --target ${{ matrix.target }} + else + cross build --release --target ${{ matrix.target }} + fi + + - name: Create release archive + run: | + mkdir -p build/bin + cp target/${{ matrix.target }}/release/flare-admin build/bin/ + cp target/${{ matrix.target }}/release/flare-stats build/bin/ + cp target/${{ matrix.target }}/release/kubectl-flare build/bin/ + cd build/bin + if [[ "${{ matrix.target }}" == *"windows"* ]]; then + zip -r flare-tools-${{ steps.tag.outputs.TAG }}-${{ matrix.target }}.zip . + else + tar -czf flare-tools-${{ steps.tag.outputs.TAG }}-${{ matrix.target }}.tar.gz . + fi + + - name: Upload Release Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./build/bin/flare-tools-${{ steps.tag.outputs.TAG }}-${{ matrix.target }}.tar.gz + asset_name: flare-tools-${{ steps.tag.outputs.TAG }}-${{ matrix.target }}.tar.gz + asset_content_type: application/gzip + + create_release: + runs-on: ubuntu-latest + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + + steps: + - uses: actions/checkout@v4 + + - name: Get tag name + id: tag + run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Create Release + uses: actions/create-release@v1 + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.tag.outputs.TAG }} + release_name: Release ${{ steps.tag.outputs.TAG }} + draft: false + prerelease: false \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..db10ea6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,35 @@ +# Goal +- Reimplement flare-tools from ruby to rust. +- Add kubectl flare command. It is a proxy command which runs flare-tools command on index server. +- Do unittest with a mock server. +- Do e2e test without a mock server. +- Use DNS names on e2e test. +- Test all commands on both tests. + +# Flare-tools commands +- Flare-admin is the main command for flare-tools. +- The commands are as follows(See ./README.txt): + - flare-admin dumpkey [hostname:port] ... + - flare-admin master [hostname:port:balance:partition] ... + - flare-admin balance [hostname:port:balance] ... + - flare-admin down [hostname:port] ... + - flare-admin restore [hostname:port] + - flare-admin stats [hostname:port] ... + - flare-admin verify + - flare-admin remove [hostname:port] ... + - flare-admin dump [hostname:port] ... + - flare-admin threads [hostname:port] + - flare-admin slave [hostname:port:balance:partition] ... + - flare-admin list + - flare-admin down [hostname:port] ... + - flare-admin ping [hostname:port] ... + - flare-admin index + - flare-admin reconstruct [hostname:port] ... + +# What is a flare? +- Flare is a memcached compatible server. (See memcached protocol from ./memacached.protocol.txt) +- Flared server is a data server. +- Flarei server is a monitoring server for failover. +- Flare has master and slave. +- Flare has shard system. +- The commands like master, slave and reconstruct calls flush_all command before the main command. diff --git a/COMMAND_MANUAL.md b/COMMAND_MANUAL.md new file mode 100644 index 0000000..dd887f0 --- /dev/null +++ b/COMMAND_MANUAL.md @@ -0,0 +1,379 @@ +# flare command manual + +## Commands compatible with memcached + +### Data operation commands + +#### get + +**syntax** + + get [key name] + +**response** + + VALUE [key name] [flag] [size of value] + (value) + END + +Get value by key name. + +#### gets + +**syntax** + + gets [key name] + +**response** + + VALUE [key name] [flag] [size of value] [version] + (value) + END + +Get value with CAS version by key name. + +#### set + +**syntax** + + set [key name] [flag] [expiration time] [size of value] + [value] + +**response** + + STORED + +- flag: the flag indicating whether or not to compress the value. +- expiration time: time [sec] to be erased. If you set this option `0`, the key will not be erased. + +Set a key unconditionally. +If the key already exists, it will be overwritten. + +#### add + +**syntax** + + add [key name] [flag] [expiration time] [size of value] + [value] + +**response** + + STORED + +Add a key. +If the key already exists, it will NOT be overwritten and abort writing. + +#### replace + +**syntax** + + replace [key name] [flag] [expiration time] [size of value] + [value] + +**response** + + STORED + +Overwrite existing key. +If the key doesn't exist, abort writing. + +#### append + +**syntax** + + append [key name] [flag] [expiration time] [size of value] + [value] + +**response** + + STORED + +Append value to existing key. +If the key doesn't exist, abort writing. + +#### prepend + +**syntax** + + prepend [key name] [flag] [expiration time] [size of value] + [value] + +**response** + + STORED + +Prepend value to existing key. +If the key doesn't exist, abort writing. + +#### incr + +**syntax** + + incr [key name] [number] + +**response** + + (incremented value) + +Increments numerical key value by given number. +If the key value is not numeric value, it will be treated `0`. + +#### decr + +**syntax** + + decr [key name] [number] + +**response** + + (decremented value) + +Decrements numerical key value by given number. +If the key value is not numeric value, the value will be `0`. + +#### delete + +**syntax** + + delete [key name] + +**response** + + DELETED + +Deletes an existing key. + +### Node operation commands + +#### version + +**syntax** + + version + +**response** + + VERSION flare-X.X.X + +Prints server version. + +#### quit + +**syntax** + + quit + +Terminate telnet session. + +### Command options + +#### noreply option + +available on set/add/replace/append/prepend/incr/decr commands, like: + + set key1 0 0 3 noreply + +If this option is specified, response will NOT be returned. + + +## Commands incompatible with memcached + +### Data operation commands + +#### flush_all + +**syntax** + + flush_all + +**response** + + OK + +Clear all data from terget node. + +#### dump + +**syntax** + + dump ([wait]) ([partition]) ([partition size]) ([bwlimit]) + +**response** + + VALUE [key] [flag] [size of value] [version] [expiration time] + (value) + ... + END + +- wait: wait microseconds for each key retrieval (default = 0) +- partition: target partition to dump +- partition size: partition size to calculate partition +- bwlimit: bandwidth limit in bytes/sec (default = no limit) + +Dump all the data in the target node. If partition arguments are specified, only data in target partition are dumped. + +#### dump_key (>= 1.1.0) + +**syntax** + + dump_key ([partition]) ([partition size]) + +**response** + + KEY [key] + ... + END + +- partition: target partition to dump +- partition size: partition size to calculate parition + +Dump all the keys in the target node. If partition arguments are specified, only keys in target partition are dumped. + +### Node operation commands + +#### kill + +**syntax** + + kill [thread id] + +**response** + + OK + +Kill specified thread (thread id is identified via "stats threads"). + +#### node add (index server only) + +**syntax** + + node add [server name] [server port] + +**response** + + OK + +Node server sends this command at startup (internal command). + +#### node role (index server only) + +**syntax** + + node role [server name] [server port] [role=(master|slave|proxy)] [balance] ([partition]) + +**response** + + OK + +Shift node role. + +#### node state (index server only) + +**syntax** + + node state [server name] [server port] [state=(active|prepare|down)] + +**response** + + OK + +Shift node state. + +#### node remove (index server only) + +**syntax** + + node remove [server name] [server port] + +Remove node from index server (only available if target node is down). + +#### node sync (index server only) + +**syntax** + + node sync + NODE [server name] [server port] [role] [state] [partition] [balance] [thread type] + ... + +**response** + + OK + +Index server sends this command to each node if index server detects any change of role or state (internal command). + +#### ping + +**syntax** + + ping + +**response** + + OK + +Always return "OK" (internal command - to watch each node). + +#### stats nodes + +**syntax** + + stats nodes + +**response** + + STAT [node key]:role [value=(master|slave|proxy)] + STAT [node_key]:state [value=(active|prepare|down)] + STAT [node_key]:partition [value] + STAT [node_key]:balance [value] + STAT [node_key]:thread_type [value] + +Show all nodes ("thread_type" is internal id). + +#### stats threads + +**syntax** + + stats threads + +**response** + + STAT [thread id]:type [value] + STAT [thread id]:peer [value] + STAT [thread id]:op [value] + STAT [thread id]:uptime [value] + STAT [thread id]:state [value] + STAT [thread id]:info [value] + STAT [thread id]:queue [value] + +Show all threads. + +### Command options + +#### sync option + +available on set/add/replace/append/prepend/incr/decr commands, like: + + set key1 0 0 3 sync + +If this option is specified, response is send ''after'' replication is done (just opposite way of "noreply" option). + +## Specification + +### key length + +Flare can accept keys more than 250 bytes. + +### value length + +Flare can accpet value more than 1M bytes (memcached returns "object too large"). + +### proxy identifier + +Flare ''internally'' add proxy identifier w/ following format (to avoid infinite loop): + + [command...] + +for example: + + get key1 + diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..00a1e09 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2604 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-config" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0baa720ebadea158c5bda642ac444a2af0cdf7bb66b46d1e4533de5d1f449d0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.3.1", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b68c2194a190e1efc999612792e25b1ab3abfefe4306494efaaabc25933c0cbe" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2090e664216c78e766b6bac10fe74d2f451c02441d43484cd76ac9a295075f7" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-ec2" +version = "1.152.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e169baec1fd3989e1c627e5522d548da98f3cc92c071ce89298b9862f0b0edf7" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssm" +version = "1.87.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d1813039b603073e01fb3cf07b095e3c5671c479f2d9f2f21ac986bef146dc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.78.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd7bc4bd34303733bded362c4c997a39130eac4310257c79aae8484b1c4b724" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.79.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77358d25f781bb106c1a69531231d4fd12c6be904edb0c47198c604df5a2dbca" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.80.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e3ed2a9b828ae7763ddaed41d51724d2661a50c45f845b08967e52f4939cfc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c82ba4cab184ea61f6edaafc1072aad3c2a17dcf4c0fce19ac5694b90d8b5f" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f108f1ca850f3feef3009bdcc977be201bca9a91058864d9de0684e64514bee0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.11", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.6.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "660f70d9d8af6876b4c9aa8dcb0dbaf0f89b04ee9a4455bea1b4ba03b15f26f6" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "937a49ecf061895fca4a6dd8e864208ed9be7546c0527d04bc07d502ec5fba1c" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cc" +version = "1.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_complete" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "flare-tools" +version = "1.0.0" +dependencies = [ + "aws-config", + "aws-sdk-ec2", + "aws-sdk-ssm", + "bytes", + "clap", + "clap_complete", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.11", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.3.1", + "hyper 1.6.0", + "hyper-util", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.2", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", + "libc", + "pin-project-lite", + "socket2 0.6.0", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets 0.53.3", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.9.4", + "windows-sys 0.60.2", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.23.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.4", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.141" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix 1.0.8", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2 0.6.0", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls 0.23.31", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b2036f5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "flare-tools" +version = "1.0.0" +edition = "2021" +authors = ["Flare Tools Contributors"] +description = "Rust implementation of flare-tools for memcached-compatible flare cluster management" +license = "MIT" + +[[bin]] +name = "flare-admin" +path = "src/bin/flare-admin.rs" + +[[bin]] +name = "flare-stats" +path = "src/bin/flare-stats.rs" + +[[bin]] +name = "kubectl-flare" +path = "src/bin/kubectl-flare.rs" + +[[bin]] +name = "flare-cluster-repl" +path = "src/bin/flare-cluster-repl.rs" + +[dependencies] +bytes = "1.5" +thiserror = "1.0" +clap = { version = "4.4", features = ["derive"] } +clap_complete = "4.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.0", features = ["full"] } +aws-config = { version = "1.0", optional = true } +aws-sdk-ssm = { version = "1.0", optional = true } +aws-sdk-ec2 = { version = "1.0", optional = true } + +[dev-dependencies] +tempfile = "3.8" + +[features] +default = [] +aws = ["aws-config", "aws-sdk-ssm", "aws-sdk-ec2"] + +[lib] +name = "flare_tools" +path = "src/lib.rs" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1cc0c31 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Multi-stage build for flare-tools (Rust) +FROM --platform=linux/amd64 rust:1.81-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git make musl-dev + +# Add musl target for static linking +RUN rustup target add x86_64-unknown-linux-musl + +# Set working directory +WORKDIR /app + +# Copy Cargo files +COPY Cargo.toml ./ + +# Copy source code +COPY src/ ./src/ +COPY tests/ ./tests/ + +# Build static binaries with musl target +ENV RUSTFLAGS='-C target-feature=+crt-static' +RUN cargo build --release --target x86_64-unknown-linux-musl + +# Final stage - minimal Alpine with static binaries +FROM --platform=linux/amd64 alpine:latest + +# Create non-root user +RUN addgroup -g 1001 flare && \ + adduser -D -s /bin/sh -u 1001 -G flare flare + +# Set working directory +WORKDIR /home/flare + +# Copy static binaries from builder stage +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/flare-admin /usr/local/bin/ +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/flare-stats /usr/local/bin/ +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/kubectl-flare /usr/local/bin/ +COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/flare-cluster-repl /usr/local/bin/ + +# Change ownership +RUN chown -R flare:flare /home/flare + +# Switch to non-root user +USER flare + +# Set default command +CMD ["flare-admin", "--help"] \ No newline at end of file diff --git a/Dockerfile.debian b/Dockerfile.debian new file mode 100644 index 0000000..7a5d044 --- /dev/null +++ b/Dockerfile.debian @@ -0,0 +1,25 @@ +FROM --platform=linux/amd64 ubuntu:noble + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + debhelper \ + golang-go \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Set Go environment +ENV GOPATH=/go +ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH + +# Create working directory +WORKDIR /build + +# Copy source code +COPY . . + +# Build the debian package +RUN dpkg-buildpackage -us -uc -b + +# List generated files +RUN ls -la / \ No newline at end of file diff --git a/README-rust.md b/README-rust.md new file mode 100644 index 0000000..539ef3a --- /dev/null +++ b/README-rust.md @@ -0,0 +1,164 @@ +# Flare Tools (Rust Implementation) + +This is a Rust implementation of flare-tools for managing memcached-compatible flare clusters. + +## Features + +- **Protocol Support**: Full implementation of memcached protocol with flare-specific extensions +- **Admin Tool**: Complete `flare-admin` CLI with all administrative commands +- **Kubernetes Integration**: `kubectl-flare` plugin for managing clusters in Kubernetes +- **Type Safety**: Leverages Rust's type system for robust protocol handling +- **Performance**: Native binary with minimal runtime overhead + +## Building + +```bash +cargo build --release +``` + +## Installation + +```bash +# Install flare-admin +cargo install --path . --bin flare-admin + +# Install kubectl-flare +cargo install --path . --bin kubectl-flare +``` + +## Usage + +### flare-admin + +Basic cluster administration: + +```bash +# Ping nodes +flare-admin ping localhost:12121 + +# Show cluster status +flare-admin stats + +# List cluster nodes +flare-admin list + +# Set node as master +flare-admin master --force server1:12121:1:0 + +# Set node as slave +flare-admin slave --force server2:12122:1:0 + +# Dump data from all master nodes +flare-admin dump --all --output backup.dump + +# Restore data +flare-admin restore server1:12121 --input backup.dump + +# Reconstruct all nodes +flare-admin reconstruct --all --force + +# Verify cluster health +flare-admin verify + +# Generate index XML +flare-admin index + +# Show thread status +flare-admin threads server1:12121 + +# Monitor cluster with flare-stats +flare-stats --qps --count 5 --wait 1 +``` + +### kubectl-flare + +For Kubernetes deployments: + +```bash +# Run flare-admin commands on the cluster +kubectl flare admin stats +kubectl flare admin list +kubectl flare admin master --force server1:12121:1:0 + +# Specify namespace and pod selector +kubectl flare -n flare-system --pod-selector app=flare-index admin stats +``` + +## Protocol Implementation + +The implementation includes: + +- **Memcached Protocol**: Complete memcached protocol parser with all standard commands +- **Flare Extensions**: + - `dump_key` - Dump keys from partitions + - `node` commands - Node management (role, state, remove, sync) + - `kill` - Thread management + - Extended `dump` with bandwidth limiting + +## Architecture + +``` +src/ +├── protocol/ +│ ├── memcached.rs # Core memcached protocol +│ └── flare.rs # Flare-specific extensions +├── client/ +│ └── mod.rs # Flare client library +├── bin/ +│ ├── flare-admin.rs # CLI administration tool +│ └── kubectl-flare.rs # Kubernetes plugin +└── lib.rs # Library exports +``` + +## Commands Supported + +All original flare-admin commands are implemented: + +- ✅ `ping` - Test node connectivity +- ✅ `stats` - Show cluster statistics +- ✅ `list` - List cluster nodes +- ✅ `master` - Configure master nodes (with flush_all) +- ✅ `slave` - Configure slave nodes (with flush_all) +- ✅ `balance` - Adjust node balance +- ✅ `down` - Mark nodes as down +- ✅ `remove` - Remove nodes from cluster +- ✅ `dump` - Export data from nodes +- ✅ `dumpkey` - Export keys from nodes +- ✅ `restore` - Import data to nodes (with VALUE parsing) +- ✅ `reconstruct` - Reconstruct node data (with flush_all) +- ✅ `verify` - Verify cluster consistency (comprehensive checks) +- ✅ `index` - Generate index XML (boost serialization format) +- ✅ `threads` - Show thread status + +Additional features: +- ✅ `flare-stats` - Dedicated statistics monitoring tool +- ✅ Dry-run mode for all destructive operations +- ✅ Force mode to skip confirmations +- ✅ Comprehensive error handling and validation + +## Testing + +```bash +# Run unit tests +cargo test + +# Run with verbose output +cargo test -- --nocapture + +# Test specific module +cargo test protocol::flare +``` + +## Development + +The codebase follows Rust best practices: + +- **Error Handling**: Uses `thiserror` for structured error types +- **CLI**: Built with `clap` for robust argument parsing +- **Async Ready**: Optional tokio support for async operations +- **Memory Safe**: No unsafe code, leverages Rust's ownership system +- **Type Safe**: Strong typing prevents protocol parsing errors + +## License + +MIT License \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f0b3ee --- /dev/null +++ b/README.md @@ -0,0 +1,394 @@ +# flare-tools (Rust Implementation) + +A Rust implementation of flare-tools, a collection of command line tools to maintain a flare cluster. + +## Overview + +This is a complete rewrite of the original Ruby-based flare-tools in Rust, providing: + +- **Performance**: Faster execution and lower memory usage +- **Safety**: Memory safety and thread safety guaranteed by Rust +- **Deployment**: Single binary deployment with no runtime dependencies +- **Maintainability**: Strong typing and comprehensive test coverage +- **Compatibility**: Full compatibility with the original Ruby implementation + +## Tools + +### flare-stats + +A command line tool for acquiring statistics of flare nodes. + +```bash +flare-stats --index-server=flare1.example.com +``` + +### flare-admin + +A command line tool for maintaining flare clusters with various subcommands: + +```bash +flare-admin [subcommand] [options] [arguments] +``` + +#### Available Subcommands + +- `ping` - Check if nodes are alive +- `stats` - Show cluster statistics +- `list` - List nodes in the cluster +- `master` - Create master partitions +- `slave` - Create slave nodes +- `balance` - Set node balance values +- `down` - Turn down nodes +- `reconstruct` - Reconstruct node databases +- `remove` - Remove nodes from cluster +- `dump` - Dump data from nodes +- `dumpkey` - Dump keys from nodes +- `restore` - Restore data to nodes +- `index` - Generate index XML +- `threads` - Show thread status +- `verify` - Verify cluster integrity + +### flare-cluster-repl + +A command line tool for configuring cluster replication settings: + +```bash +flare-cluster-repl [environment] [config-file] +``` + +Supports both Docker Compose and AWS environments for updating flared.conf files with replication settings. + +## Installation + +### Pre-built Binaries + +Download the latest binaries from the [releases page](https://github.com/gree/flare-tools/releases). + +### From Source + +```bash +# Clone the repository +git clone https://github.com/gree/flare-tools.git +cd flare-tools + +# Build and install +cargo build --release +cargo install --path . +``` + +### Using Cargo + +```bash +cargo install flare-tools +``` + +### Docker + +```bash +docker build -t flare-tools . +docker run --rm flare-tools flare-admin --help +``` + +## Configuration + +### Environment Variables + +- `FLARE_INDEX_SERVER` - Index server hostname or hostname:port +- `FLARE_INDEX_SERVER_PORT` - Index server port (default: 12120) + +### Command Line Options + +Common options available for all commands: + +- `--index-server` - Index server hostname +- `--index-server-port` - Index server port +- `--debug` - Enable debug mode +- `--warn` - Turn on warnings +- `--dry-run` - Dry run mode (flare-admin only) +- `--force` - Skip confirmation prompts +- `--help` - Show help message + +## Shell Completion + +Flare-admin supports shell completion for commands, options, and node names. This makes it easier to use the tool interactively. + +### Installing Completion + +1. Build the flare-admin binary: + ```bash + cargo build --bin flare-admin + ``` + +2. Install completion for your shell: + ```bash + # For bash + ./install-completion.sh bash + + # For zsh + ./install-completion.sh zsh + + # For fish + ./install-completion.sh fish + + # Or generate manually for any supported shell + ./target/debug/flare-admin completion bash > /etc/bash_completion.d/flare-admin + ``` + +3. Restart your shell or source the completion file. + +### Using Completion + +Once installed, you can use tab completion for: + +- **Commands**: `flare-admin ` shows all available commands +- **Options**: `flare-admin --` shows all available options +- **Subcommand options**: `flare-admin ping --` shows options for ping command +- **Node names**: Commands that accept node addresses will show actual cluster nodes +- **Node specifications**: Commands like `master` and `slave` complete with balance:partition formats + +### Supported Shells + +- Bash +- Zsh +- Fish +- Elvish +- PowerShell + +### Examples + +```bash +# Command completion +flare-admin +# Shows: ping stats list master slave balance down remove dump dumpkey restore reconstruct verify index threads activate completion + +# Option completion +flare-admin -- +# Shows: --index-server --index-port --force --dry-run --help --version + +# Node name completion +flare-admin ping +# Shows: localhost:12121 flare-prod-master-1:12121 flare-prod-master-2:12121 flare-prod-slave-1:12121 flare-prod-slave-2:12121 + +# Node specification completion (for master/slave commands) +flare-admin master +# Shows: localhost:12121:1:0 localhost:12121:1:1 flare-prod-master-1:12121:1:0 ... +``` + +## Usage Examples + +### Basic Statistics + +```bash +# Show cluster statistics +flare-stats --index-server=flare1.example.com + +# Show statistics with QPS information +flare-stats --index-server=flare1.example.com --qps + +# Repeat statistics every 5 seconds, 10 times +flare-stats --index-server=flare1.example.com --wait=5 --count=10 +``` + +### Cluster Management + +```bash +# Ping nodes +flare-admin ping --index-server=flare1.example.com + +# List nodes in cluster +flare-admin list --index-server=flare1.example.com + +# Create master partition +flare-admin master --index-server=flare1.example.com newmaster:12131:1:1 + +# Create slave nodes +flare-admin slave --index-server=flare1.example.com newslave:12132:1:0 + +# Set node balance +flare-admin balance --index-server=flare1.example.com node1:12131:3 +``` + +### Data Operations + +```bash +# Dump data from all master nodes +flare-admin dump --index-server=flare1.example.com --all --output=backup.data + +# Restore data to node +flare-admin restore --index-server=flare1.example.com --input=backup.data node1:12131 +``` + +## Development + +### Prerequisites + +- Rust 1.75 or later +- Docker (for integration/e2e tests) +- Docker Compose + +### Building + +```bash +# Build binaries +cargo build --release + +# Run tests +cargo test + +# Run all tests with integration tests +cargo test --all-features + +# Generate coverage report +cargo tarpaulin +``` + +### Testing + +**Important**: Tests run with single thread by default (configured in `.cargo/config.toml`) to prevent Docker container conflicts when running integration tests. + +#### Unit Tests + +```bash +cargo test +``` + +#### Integration Tests + +```bash +# Run flare-admin integration tests +cargo test flare_admin_integration_test -- --nocapture + +# Run flare-cluster-repl integration tests +cargo test flare_cluster_repl_integration_test -- --nocapture +``` + +#### E2E Testing Setup + +Before running e2e tests, you need to set up your environment: + +1. **Update /etc/hosts** - Add these entries to `/etc/hosts`: + ``` + # Single cluster setup + 127.0.0.1 flarei + 127.0.0.1 flared1 + 127.0.0.1 flared2 + 127.0.0.1 flared3 + 127.0.0.1 flared4 + + # Multi-cluster setup (for flare-cluster-repl testing) + 127.0.0.1 flarei-prod + 127.0.0.1 flare-prod-master-1 + 127.0.0.1 flare-prod-master-2 + 127.0.0.1 flare-prod-slave-1 + 127.0.0.1 flare-prod-slave-2 + 127.0.0.1 flarei-staging + 127.0.0.1 flare-staging-master-1 + 127.0.0.1 flare-staging-slave-1 + ``` + +2. **Start test clusters**: + ```bash + # For single cluster tests + docker-compose up -d + + # For multi-cluster tests + docker-compose -f docker-compose-multi-cluster.yml up -d + ``` + +3. **Run e2e tests**: + ```bash + # Test flare-admin with single cluster + cargo test flare_admin_integration_test -- --nocapture + + # Test flare-cluster-repl with multi-cluster + cargo test flare_cluster_repl_integration_test -- --nocapture + ``` + +4. **Clean up**: + ```bash + # Stop single cluster + docker-compose down + + # Stop multi-cluster + docker-compose -f docker-compose-multi-cluster.yml down + ``` + +#### Test Environment Details + +- **Unit tests**: Mock server implementation, no external dependencies +- **Integration tests**: Real flare cluster using Docker containers with DNS names +- **E2E tests**: Full cluster topology with master/slave configuration and replication testing + +### Code Quality + +```bash +# Format code +cargo fmt + +# Lint code +cargo clippy + +# Check code +cargo check +``` + +## Project Structure + +``` +. +├── src/ +│ ├── bin/ # Command line applications +│ │ ├── flare-admin.rs # flare-admin command +│ │ ├── flare-stats.rs # flare-stats command +│ │ └── flare-cluster-repl.rs # flare-cluster-repl command +│ ├── lib.rs # Library root +│ └── modules/ # Internal modules +├── tests/ # Integration tests +│ ├── flare_admin_integration_test.rs +│ └── flare_cluster_repl_integration_test.rs +├── examples/ # Configuration examples +├── docker-compose.yml # Single cluster setup +├── docker-compose-multi-cluster.yml # Multi-cluster setup +├── Dockerfile # Docker configuration +├── Cargo.toml # Rust project configuration +└── Cargo.lock # Dependency lockfile +``` + +## Migration from Ruby Version + +This Rust implementation maintains full compatibility with the original Ruby version: + +- All command line options are preserved +- Output formats are identical +- Environment variable support is maintained +- All subcommands and their behaviors are replicated + +### Key Improvements + +1. **Performance**: Significantly faster startup and execution +2. **Safety**: Memory safety and thread safety guaranteed by Rust +3. **Memory Usage**: Lower memory footprint +4. **Deployment**: Single binary with no runtime dependencies +5. **Error Handling**: More robust error handling and reporting +6. **Testing**: Comprehensive test coverage including e2e tests +7. **Maintenance**: Easier to maintain and extend + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests for new functionality +5. Run the test suite +6. Submit a pull request + +## License + +MIT-style license - see LICENSE file for details. + +## Authors + +- Original Ruby implementation: Kiyoshi Ikehara +- Rust implementation: Converted from Ruby with full compatibility + +Copyright (C) GREE, Inc. 2011-2024. \ No newline at end of file diff --git a/cluster-config/flared1.conf b/cluster-config/flared1.conf new file mode 100644 index 0000000..ff7fcd2 --- /dev/null +++ b/cluster-config/flared1.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flared1 +index-servers = flarei:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flared1 +server-port = 12121 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/flared2.conf b/cluster-config/flared2.conf new file mode 100644 index 0000000..377ee1a --- /dev/null +++ b/cluster-config/flared2.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flared2 +index-servers = flarei:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flared2 +server-port = 12122 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/flared3.conf b/cluster-config/flared3.conf new file mode 100644 index 0000000..0843483 --- /dev/null +++ b/cluster-config/flared3.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flared3 +index-servers = flarei:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flared3 +server-port = 12123 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/flared4.conf b/cluster-config/flared4.conf new file mode 100644 index 0000000..f032b4d --- /dev/null +++ b/cluster-config/flared4.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flared4 +index-servers = flarei:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flared4 +server-port = 12124 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/flarei.conf b/cluster-config/flarei.conf new file mode 100644 index 0000000..bb280cf --- /dev/null +++ b/cluster-config/flarei.conf @@ -0,0 +1,8 @@ +data-dir = /tmp/flarei +log-facility = local0 +max-connection = 256 +monitor-threshold = 3 +monitor-interval = 1 +server-name = flarei +server-port = 12120 +thread-pool-size = 8 diff --git a/cluster-config/production/flare-prod-master-1.conf b/cluster-config/production/flare-prod-master-1.conf new file mode 100644 index 0000000..b36decc --- /dev/null +++ b/cluster-config/production/flare-prod-master-1.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-prod-master-1 +index-servers = flarei-prod:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-prod-master-1 +server-port = 12121 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/production/flare-prod-master-2.conf b/cluster-config/production/flare-prod-master-2.conf new file mode 100644 index 0000000..b53b8b4 --- /dev/null +++ b/cluster-config/production/flare-prod-master-2.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-prod-master-2 +index-servers = flarei-prod:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-prod-master-2 +server-port = 12122 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/production/flare-prod-slave-1.conf b/cluster-config/production/flare-prod-slave-1.conf new file mode 100644 index 0000000..a202889 --- /dev/null +++ b/cluster-config/production/flare-prod-slave-1.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-prod-slave-1 +index-servers = flarei-prod:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-prod-slave-1 +server-port = 12123 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/production/flare-prod-slave-2.conf b/cluster-config/production/flare-prod-slave-2.conf new file mode 100644 index 0000000..febdc54 --- /dev/null +++ b/cluster-config/production/flare-prod-slave-2.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-prod-slave-2 +index-servers = flarei-prod:12120 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-prod-slave-2 +server-port = 12124 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/production/flarei-prod.conf b/cluster-config/production/flarei-prod.conf new file mode 100644 index 0000000..5989d13 --- /dev/null +++ b/cluster-config/production/flarei-prod.conf @@ -0,0 +1,6 @@ +data-dir = /tmp/flarei-prod +log-facility = local1 +max-connection = 128 +server-name = flarei-prod +server-port = 12120 +thread-pool-size = 8 diff --git a/cluster-config/staging/flare-staging-master-1.conf b/cluster-config/staging/flare-staging-master-1.conf new file mode 100644 index 0000000..2f2f2e5 --- /dev/null +++ b/cluster-config/staging/flare-staging-master-1.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-staging-master-1 +index-servers = flarei-staging:12130 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-staging-master-1 +server-port = 12131 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/staging/flare-staging-slave-1.conf b/cluster-config/staging/flare-staging-slave-1.conf new file mode 100644 index 0000000..abe30cb --- /dev/null +++ b/cluster-config/staging/flare-staging-slave-1.conf @@ -0,0 +1,10 @@ +data-dir = /tmp/flare-staging-slave-1 +index-servers = flarei-staging:12130 +log-facility = local1 +max-connection = 256 +mutex-slot = 32 +proxy-concurrency = 2 +server-name = flare-staging-slave-1 +server-port = 12132 +storage-type = tch +thread-pool-size = 16 diff --git a/cluster-config/staging/flarei-staging.conf b/cluster-config/staging/flarei-staging.conf new file mode 100644 index 0000000..3bf2736 --- /dev/null +++ b/cluster-config/staging/flarei-staging.conf @@ -0,0 +1,6 @@ +data-dir = /tmp/flarei-staging +log-facility = local1 +max-connection = 128 +server-name = flarei-staging +server-port = 12130 +thread-pool-size = 8 diff --git a/debian-packages/README.md b/debian-packages/README.md new file mode 100644 index 0000000..5b551ab --- /dev/null +++ b/debian-packages/README.md @@ -0,0 +1,135 @@ +# flare +flare is a distributed, and persistent key-value storage compatible with [memcached](http://memcached.org/), with several additional features (as follows): + +- persistent storage (you can use flare as persistent memcached) +- pluggable storage + - [Tokyo Cabinet](http://fallabs.com/tokyocabinet/) + - [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) (experimental) +- data replication (synchronous or asynchronous) +- data partitioning (automatically partitioned according to the number of master servers (transparent for clients) +- dynamic reconstruction, and partitioning (you can dynamically (I mean, without any service interruption) add slave servers and partition master servers) +- node monitoring and failover (if any server is down, the server is automatically isolated from active servers and another slave server is promoted to master server) +- request proxy (you can always get same result regardless of servers you connect to, so you can think of a flare cluster as one big key-value storage) +- over 256 bytes keys, and over 1M bytes values are available + +flare is free software base on [GNU GENERAL PUBLIC LICENSE Version 2](http://www.gnu.org/licenses/gpl-2.0.html). + +## Supported Operating Systems +flare is mainly developed under following platforms: + +- Debian GNU/Linux (etch or later, both i386 and amd64) +- Mac OS X (Darwin 9.5.0, i386, amd64) +- FreeBSD +- other UNIX like OSs. + +## Dependent library +### Run-time +- [boost](http://www.boost.org/) +- [Tokyo Cabinet](http://fallabs.com/tokyocabinet/) +- [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) (optional) +- zlib +- libhashkit +- uuid + +### Build-time +- [gcc](https://gcc.gnu.org/) +- autoconf +- automake +- libtool + +## Install from source code on Ubuntu 14.04 (Trusty Tahr) +### Installation of depending packages +First, install depending packages by `apt-get`. +``` +$ sudo apt-get install \ + git \ + locales \ + zlib1g-dev \ + build-essential \ + autoconf \ + automake \ + libtool \ + libboost-all-dev \ + libhashkit-dev \ + libtokyocabinet-dev \ + uuid-dev +``` + +### Installation of flare +Download source code, and compile it. +``` +$ git clone https://github.com/gree/flare.git +$ cd flare +$ ./autogen.sh +$ ./configure +$ make +$ make check +$ sudo make install +``` +If you want to optional packages, you should run `./configure` with options. +**You can see available options by `./configure --help`.** + +#### For example (when use Kyoto Cabinet): +First, you must install `libkyotocabinet-dev` in addition to depending packages. +``` +$ sudo apt-get install libkyotocabinet-dev +``` +And run `./configure` with `--with-kyotocabinet` option. +``` +$ ./configure --with-kyotocabinet=/usr/include +``` + +## Install flare via Nix on Ubuntu or MacOS + +We experimentally support development with nix package manager. + +### Installation of flare + +Just type following commands. + +``` +# Install nix package manager +$ curl -L https://nixos.org/nix/install | sh +# Install flare +$ nix profile install github:gree/flare +``` + +### Development of flare via Nix + +``` +# Clone source code +$ git clone git@github.com:gree/flare.git + +# Enter a development environment of nix +$ nix develop + +# Build source codes +$ ./autogen.sh +$ ./configure +$ make +$ make test +``` + +## Build Debian Package with Docker + +```bash +# Build Debian package +$ ./build-debian-docker.sh + +# Install the package +$ sudo dpkg -i debian-packages/kvs-flare*.deb +``` + +## Create configuration file +Copy default configuration files from `etc`, and modify it. +``` +$ sudo cp etc/flarei.conf /etc/ +$ sudo cp etc/flared.conf /etc/ +``` + +## Run +Now, you can run flare. +``` +$ sudo /usr/local/bin/flarei -f /etc/flarei.conf --daemonize +$ sudo /usr/local/bin/flared -f /etc/flared.conf --daemonize +``` diff --git a/debian-packages/test/mock-flare-cluster/main.go b/debian-packages/test/mock-flare-cluster/main.go new file mode 100644 index 0000000..fe02bf3 --- /dev/null +++ b/debian-packages/test/mock-flare-cluster/main.go @@ -0,0 +1,316 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "net" + "strconv" + "strings" + "sync" +) + +type NodeState string +type NodeRole string + +const ( + StateActive NodeState = "active" + StateDown NodeState = "down" + StateProxy NodeState = "proxy" + StateReady NodeState = "ready" + + RoleMaster NodeRole = "master" + RoleSlave NodeRole = "slave" + RoleProxy NodeRole = "proxy" +) + +type Node struct { + Host string + Port int + Role NodeRole + State NodeState + Partition int + Balance int + Items int64 + Conn int + Behind int64 + Hit float64 + Size int64 + Uptime string + Version string + QPS float64 + QPSR float64 + QPSW float64 +} + +type MockFlareCluster struct { + nodes map[string]*Node + mutex sync.RWMutex +} + +func NewMockFlareCluster() *MockFlareCluster { + cluster := &MockFlareCluster{ + nodes: make(map[string]*Node), + } + + cluster.initializeCluster() + return cluster +} + +func (c *MockFlareCluster) initializeCluster() { + nodes := []*Node{ + { + Host: "127.0.0.1", Port: 12121, Role: RoleMaster, State: StateActive, + Partition: 0, Balance: 1, Items: 10000, Conn: 50, Behind: 0, + Hit: 95.5, Size: 1024, Uptime: "2d", Version: "1.3.4", + QPS: 150.5, QPSR: 80.2, QPSW: 70.3, + }, + { + Host: "127.0.0.1", Port: 12122, Role: RoleMaster, State: StateActive, + Partition: 1, Balance: 1, Items: 10001, Conn: 55, Behind: 0, + Hit: 94.8, Size: 1025, Uptime: "2d", Version: "1.3.4", + QPS: 145.8, QPSR: 75.5, QPSW: 70.3, + }, + { + Host: "127.0.0.1", Port: 12123, Role: RoleSlave, State: StateActive, + Partition: 0, Balance: 1, Items: 10000, Conn: 30, Behind: 5, + Hit: 0.0, Size: 1024, Uptime: "2d", Version: "1.3.4", + QPS: 80.2, QPSR: 80.2, QPSW: 0.0, + }, + { + Host: "127.0.0.1", Port: 12124, Role: RoleSlave, State: StateActive, + Partition: 1, Balance: 1, Items: 10001, Conn: 32, Behind: 3, + Hit: 0.0, Size: 1025, Uptime: "2d", Version: "1.3.4", + QPS: 82.1, QPSR: 82.1, QPSW: 0.0, + }, + } + + for _, node := range nodes { + key := fmt.Sprintf("%s:%d", node.Host, node.Port) + c.nodes[key] = node + } +} + +func (c *MockFlareCluster) handleConnection(conn net.Conn) { + defer conn.Close() + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + command := strings.TrimSpace(scanner.Text()) + log.Printf("Received command: %s", command) + + response := c.processCommand(command) + conn.Write([]byte(response)) + } +} + +func (c *MockFlareCluster) processCommand(command string) string { + parts := strings.Fields(command) + if len(parts) == 0 { + return "ERROR invalid command\r\nEND\r\n" + } + + cmd := strings.ToLower(parts[0]) + + switch cmd { + case "ping": + return "OK\r\nEND\r\n" + case "stats": + return c.getStats() + case "node_add": + return c.handleNodeAdd(parts[1:]) + case "node_role": + return c.handleNodeRole(parts[1:]) + case "node_state": + return c.handleNodeState(parts[1:]) + case "node_remove": + return c.handleNodeRemove(parts[1:]) + case "node_balance": + return c.handleNodeBalance(parts[1:]) + case "threads": + return c.getThreads() + case "version": + return "VERSION 1.3.4\r\nEND\r\n" + default: + return "ERROR unknown command\r\nEND\r\n" + } +} + +func (c *MockFlareCluster) getStats() string { + c.mutex.RLock() + defer c.mutex.RUnlock() + + var stats strings.Builder + + for _, node := range c.nodes { + line := fmt.Sprintf("%s:%d %s %s %d %d %d %d %d %.1f %d %s %s %.1f %.1f %.1f\r\n", + node.Host, node.Port, node.State, node.Role, node.Partition, node.Balance, + node.Items, node.Conn, node.Behind, node.Hit, node.Size, node.Uptime, + node.Version, node.QPS, node.QPSR, node.QPSW) + stats.WriteString(line) + } + + stats.WriteString("END\r\n") + return stats.String() +} + +func (c *MockFlareCluster) handleNodeAdd(args []string) string { + if len(args) < 4 { + return "ERROR insufficient arguments\r\nEND\r\n" + } + + hostPort := args[0] + role := args[1] + partition, _ := strconv.Atoi(args[2]) + balance, _ := strconv.Atoi(args[3]) + + parts := strings.Split(hostPort, ":") + if len(parts) != 2 { + return "ERROR invalid host:port format\r\nEND\r\n" + } + + port, err := strconv.Atoi(parts[1]) + if err != nil { + return "ERROR invalid port\r\nEND\r\n" + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + node := &Node{ + Host: parts[0], Port: port, Role: NodeRole(role), State: StateReady, + Partition: partition, Balance: balance, Items: 0, Conn: 0, Behind: 0, + Hit: 0.0, Size: 0, Uptime: "0s", Version: "1.3.4", + QPS: 0.0, QPSR: 0.0, QPSW: 0.0, + } + + key := fmt.Sprintf("%s:%d", node.Host, node.Port) + c.nodes[key] = node + + return "OK\r\nEND\r\n" +} + +func (c *MockFlareCluster) handleNodeRole(args []string) string { + if len(args) < 2 { + return "ERROR insufficient arguments\r\nEND\r\n" + } + + hostPort := args[0] + role := args[1] + + c.mutex.Lock() + defer c.mutex.Unlock() + + if node, exists := c.nodes[hostPort]; exists { + node.Role = NodeRole(role) + if role == "master" { + node.State = StateActive + node.Items = 10000 + node.QPS = 150.0 + node.QPSR = 75.0 + node.QPSW = 75.0 + } else if role == "slave" { + node.State = StateActive + node.Items = 10000 + node.QPS = 80.0 + node.QPSR = 80.0 + node.QPSW = 0.0 + } + return "OK\r\nEND\r\n" + } + + return "ERROR node not found\r\nEND\r\n" +} + +func (c *MockFlareCluster) handleNodeState(args []string) string { + if len(args) < 2 { + return "ERROR insufficient arguments\r\nEND\r\n" + } + + hostPort := args[0] + state := args[1] + + c.mutex.Lock() + defer c.mutex.Unlock() + + if node, exists := c.nodes[hostPort]; exists { + node.State = NodeState(state) + return "OK\r\nEND\r\n" + } + + return "ERROR node not found\r\nEND\r\n" +} + +func (c *MockFlareCluster) handleNodeRemove(args []string) string { + if len(args) < 1 { + return "ERROR insufficient arguments\r\nEND\r\n" + } + + hostPort := args[0] + + c.mutex.Lock() + defer c.mutex.Unlock() + + if _, exists := c.nodes[hostPort]; exists { + delete(c.nodes, hostPort) + return "OK\r\nEND\r\n" + } + + return "ERROR node not found\r\nEND\r\n" +} + +func (c *MockFlareCluster) handleNodeBalance(args []string) string { + if len(args) < 2 { + return "ERROR insufficient arguments\r\nEND\r\n" + } + + hostPort := args[0] + balance, err := strconv.Atoi(args[1]) + if err != nil { + return "ERROR invalid balance value\r\nEND\r\n" + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + if node, exists := c.nodes[hostPort]; exists { + node.Balance = balance + return "OK\r\nEND\r\n" + } + + return "ERROR node not found\r\nEND\r\n" +} + +func (c *MockFlareCluster) getThreads() string { + return "thread_pool_size=16\r\nactive_threads=8\r\nqueue_size=0\r\nEND\r\n" +} + +func main() { + port := flag.Int("port", 12120, "Port to listen on") + flag.Parse() + + cluster := NewMockFlareCluster() + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatal("Failed to listen:", err) + } + defer listener.Close() + + log.Printf("Mock Flare cluster listening on port %d", *port) + log.Println("Initialized with 2 masters and 2 slaves:") + for key, node := range cluster.nodes { + log.Printf(" %s: %s %s (partition %d)", key, node.Role, node.State, node.Partition) + } + + for { + conn, err := listener.Accept() + if err != nil { + log.Printf("Failed to accept connection: %v", err) + continue + } + + go cluster.handleConnection(conn) + } +} diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..f1edf8c --- /dev/null +++ b/debian/changelog @@ -0,0 +1,11 @@ +flare-tools (1.0.0-1) noble; urgency=medium + + * Initial Go implementation of flare-tools + * Convert from Ruby to Go with full protocol compatibility + * Add flare-admin for cluster management operations + * Add flare-stats for monitoring cluster statistics + * Support all administrative commands (master, slave, reconstruct, etc.) + * Add comprehensive unit tests and e2e tests + * Add dry-run support for destructive operations + + -- Junji Hashimoto Thu, 04 Jul 2025 00:00:00 +0000 \ No newline at end of file diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..9a03714 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..f0975ea --- /dev/null +++ b/debian/control @@ -0,0 +1,16 @@ +Source: flare-tools +Section: database +Priority: optional +Maintainer: Junji Hashimoto +Build-Depends: debhelper (>= 10), golang-go (>= 1.18) +Standards-Version: 4.5.1 +Homepage: https://github.com/gree/flare-tools + +Package: flare-tools +Architecture: amd64 +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: Command line tools for flare distributed key-value storage + Flare-tools provides administrative and monitoring utilities for flare, + a distributed key-value storage system. It includes flare-admin for + cluster management operations and flare-stats for monitoring cluster + statistics and node information. \ No newline at end of file diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..145820c --- /dev/null +++ b/debian/copyright @@ -0,0 +1,26 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: flare-tools +Source: https://github.com/gree/flare-tools + +Files: * +Copyright: 2025 Junji Hashimoto +License: MIT + +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. \ No newline at end of file diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..155c11a --- /dev/null +++ b/debian/install @@ -0,0 +1,2 @@ +flare-admin usr/bin +flare-stats usr/bin \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..c770541 --- /dev/null +++ b/debian/rules @@ -0,0 +1,20 @@ +#!/usr/bin/make -f + +%: + dh $@ + +override_dh_auto_build: + go build -o flare-admin ./cmd/flare-admin + go build -o flare-stats ./cmd/flare-stats + +override_dh_auto_install: + mkdir -p debian/flare-tools/usr/bin + cp flare-admin debian/flare-tools/usr/bin/ + cp flare-stats debian/flare-tools/usr/bin/ + +override_dh_auto_clean: + rm -f flare-admin flare-stats + dh_auto_clean + +override_dh_auto_test: + # Skip tests during package build - they require network connections \ No newline at end of file diff --git a/docker-compose-multi-cluster.yml b/docker-compose-multi-cluster.yml new file mode 100644 index 0000000..a02a4da --- /dev/null +++ b/docker-compose-multi-cluster.yml @@ -0,0 +1,135 @@ +version: '3.8' + +services: + # Production Cluster - Index Server + flarei-prod: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flarei + platform: linux/amd64 + container_name: flarei-prod + ports: + - "12120:12120" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flarei-prod && cp /config/production/flarei-prod.conf /etc/flarei.conf && /usr/bin/flarei -f /etc/flarei.conf -s"] + + # Production Cluster - Master 1 + flare-prod-master-1: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-prod-master-1 + ports: + - "12121:12121" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-prod-master-1 && cp /config/production/flare-prod-master-1.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-prod + + # Production Cluster - Master 2 + flare-prod-master-2: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-prod-master-2 + ports: + - "12122:12122" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-prod-master-2 && cp /config/production/flare-prod-master-2.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-prod + + # Production Cluster - Slave 1 + flare-prod-slave-1: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-prod-slave-1 + ports: + - "12123:12123" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-prod-slave-1 && cp /config/production/flare-prod-slave-1.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-prod + + # Production Cluster - Slave 2 + flare-prod-slave-2: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-prod-slave-2 + ports: + - "12124:12124" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-prod-slave-2 && cp /config/production/flare-prod-slave-2.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-prod + + # Staging Cluster - Index Server + flarei-staging: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flarei + platform: linux/amd64 + container_name: flarei-staging + ports: + - "12130:12130" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flarei-staging && cp /config/staging/flarei-staging.conf /etc/flarei.conf && /usr/bin/flarei -f /etc/flarei.conf -s"] + + # Staging Cluster - Master 1 + flare-staging-master-1: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-staging-master-1 + ports: + - "12131:12131" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-staging-master-1 && cp /config/staging/flare-staging-master-1.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-staging + + # Staging Cluster - Slave 1 + flare-staging-slave-1: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flare-staging-slave-1 + ports: + - "12132:12131" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flare-staging-slave-1 && cp /config/staging/flare-staging-slave-1.conf /etc/flared.conf && /usr/bin/flared -f /etc/flared.conf -s"] + depends_on: + - flarei-staging + + # Flare Cluster Replication Tool + flare-cluster-repl: + build: . + platform: linux/amd64 + container_name: flare-cluster-repl + volumes: + - ./examples:/app/examples:ro + - /var/run/docker.sock:/var/run/docker.sock + - ./logs:/app/logs + working_dir: /app + entrypoint: ["flare-cluster-repl"] + command: ["--help"] + user: root # Need root to access docker socket + +volumes: + logs: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..acf2dd7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,77 @@ +version: '3.8' + +services: + # Flare Index Server + flarei: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flarei + platform: linux/amd64 + container_name: flarei + ports: + - "12120:12120" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flarei && /usr/bin/flarei -f /config/flarei.conf -s"] + depends_on: [] + + # Flare Data Server 1 (Master) + flared1: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flared1 + ports: + - "12121:12121" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flared1 && /usr/bin/flared -f /config/flared1.conf -s"] + depends_on: + - flarei + + # Flare Data Server 2 (Master) + flared2: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flared2 + ports: + - "12122:12122" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flared2 && /usr/bin/flared -f /config/flared2.conf -s"] + depends_on: + - flarei + + # Flare Data Server 3 (Slave) + flared3: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flared3 + ports: + - "12123:12123" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flared3 && /usr/bin/flared -f /config/flared3.conf -s"] + depends_on: + - flarei + + # Flare Data Server 4 (Slave) + flared4: + build: + context: ./flare-cluster-k8s + dockerfile: dockerfiles/flared + platform: linux/amd64 + container_name: flared4 + ports: + - "12124:12124" + volumes: + - ./cluster-config:/config + command: ["bash", "-c", "mkdir -p /tmp/flared4 && /usr/bin/flared -f /config/flared4.conf -s"] + depends_on: + - flarei + diff --git a/docs/dump-restore-protocol.md b/docs/dump-restore-protocol.md new file mode 100644 index 0000000..de7a50b --- /dev/null +++ b/docs/dump-restore-protocol.md @@ -0,0 +1,107 @@ +# Flare Dump/Restore Protocol + +## Dump Command Protocol + +The `dump` command retrieves all key-value pairs from flare nodes. + +### Request Format +``` +dump\r\n +``` +or +``` +dump \r\n +``` + +### Response Format +``` +VALUE \r\n +\r\n +VALUE \r\n +\r\n +... +END\r\n +``` + +Example: +``` +VALUE user:1 0 10\r\n +john_doe\r\n +VALUE user:2 0 11\r\n +jane_smith\r\n +END\r\n +``` + +## Dumpkey Command Protocol + +The `dump_key` command retrieves all keys (without values) from flare nodes. + +### Request Format +``` +dump_key\r\n +``` +or +``` +dump_key \r\n +``` + +### Response Format +``` +KEY \r\n +KEY \r\n +... +END\r\n +``` + +Example: +``` +KEY user:1\r\n +KEY user:2\r\n +KEY session:abc\r\n +END\r\n +``` + +## Restore Command Protocol + +The `restore` command sends data back to flare nodes using SET commands. + +### Request Format +For each key-value pair: +``` +set \r\n +\r\n +``` + +### Response Format +For each SET command: +``` +STORED\r\n +``` +or +``` +SERVER_ERROR \r\n +``` + +### Dump File Format for Restore + +The dump file format expected by restore command: +``` +VALUE \r\n +\r\n +VALUE \r\n +\r\n +... +``` + +Note: The END marker is not included in dump files for restore. + +## Implementation Notes + +1. The dump command should handle large datasets by streaming results +2. The restore command should: + - Parse VALUE lines from dump files + - Convert them to SET commands (with exptime=0) + - Handle errors gracefully + - Support filtering (include/exclude patterns) +3. Binary data should be handled correctly +4. Line endings must be \r\n (CRLF) as per memcached protocol \ No newline at end of file diff --git a/docs/e2e-testing.md b/docs/e2e-testing.md new file mode 100644 index 0000000..d395ba3 --- /dev/null +++ b/docs/e2e-testing.md @@ -0,0 +1,264 @@ +# E2E Testing Guide for Flare Tools + +This document describes how to run end-to-end (e2e) tests for the Go implementation of flare-tools. + +## Overview + +There are two types of e2e tests available: + +1. **Mock Server Tests** - Run against a mock flare server (fast, isolated) +2. **Kubernetes Cluster Tests** - Run against a real flare cluster in Kubernetes (comprehensive, realistic) + +## Prerequisites + +### For Mock Server Tests +- Go 1.21 or later +- No external dependencies + +### For Kubernetes Cluster Tests +- Kubernetes cluster with flare deployed +- kubectl configured to access the cluster +- Docker (for building Linux binaries) + +## Running Mock Server Tests + +These tests use a mock flare server and test basic command functionality: + +```bash +# Run all mock-based e2e tests +go test -v ./test/e2e + +# Run specific test +go test -v ./test/e2e -run TestFlareStatsE2E + +# Run with race detection +go test -race -v ./test/e2e +``` + +### Mock Test Coverage + +The mock tests cover: +- ✅ flare-stats basic functionality +- ✅ flare-stats with QPS +- ✅ flare-admin ping +- ✅ flare-admin stats +- ✅ flare-admin list +- ✅ Help commands +- ✅ Error handling +- ✅ Environment variables +- ⚠️ Master/Slave/Reconstruct (limited due to flush_all requirements) + +## Running Kubernetes Cluster Tests + +These tests run against a real flare cluster and provide comprehensive validation. + +### Step 1: Deploy Flare Cluster + +```bash +# Deploy flare cluster using kustomize +kubectl apply -k flare-cluster-k8s/base + +# Wait for pods to be ready +kubectl get pods -w +``` + +### Step 2: Build and Copy Binaries + +```bash +# Build Linux binaries +make build-linux +# or manually: +GOOS=linux GOARCH=amd64 go build -o build/flare-admin-linux cmd/flare-admin/main.go +GOOS=linux GOARCH=amd64 go build -o build/flare-stats-linux cmd/flare-stats/main.go + +# Copy binaries to cluster nodes +./scripts/copy-to-e2e.sh +``` + +### Step 3: Run Comprehensive E2E Tests + +```bash +# Run all e2e tests on Kubernetes cluster +./scripts/k8s-e2e-test.sh +``` + +### Step 4: Run Individual Tests + +You can also run individual commands manually: + +```bash +# Test list command +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 list + +# Test stats command +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 stats + +# Test flare-stats +kubectl exec node-0 -- /usr/local/bin/flare-stats -i flarei.default.svc.cluster.local -p 13300 + +# Test reconstruct command +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 reconstruct --force node-2.flared.default.svc.cluster.local:13301 +``` + +## Test Script Details + +### scripts/k8s-e2e-test.sh + +This comprehensive test script covers: + +1. **Basic Operations** + - `list` - Display cluster topology + - `stats` - Show cluster statistics + - `ping` - Test connectivity + +2. **flare-stats Tool** + - Basic stats display + - QPS (queries per second) metrics + +3. **Administrative Commands** + - `master` - Promote proxy to master + - `slave` - Convert proxy to slave + - `balance` - Adjust node balance + - `down` - Take node down + - `activate` - Bring node back up + - `reconstruct` - Rebuild node database + +4. **Advanced Features** + - Environment variable configuration + - Help command validation + - Dry-run operations + +### scripts/copy-to-e2e.sh + +This script: +- Builds Linux binaries if needed +- Copies binaries to all cluster pods +- Sets proper permissions +- Tests basic functionality + +## Expected Test Results + +### Successful Test Output + +``` +=== Running E2E tests on Kubernetes flare cluster === + +Test 1: List nodes +node partition role state balance +node-0.flared.default.svc.cluster.local:13301 1 master active 1 +node-1.flared.default.svc.cluster.local:13301 0 master active 1 +node-2.flared.default.svc.cluster.local:13301 1 slave active 1 + +Test 3: Ping +alive: flarei.default.svc.cluster.local:13300 + +Test 11: Reconstruct command +Reconstructing nodes... +reconstructing node (node=node-2.flared.default.svc.cluster.local:13301, role=slave) +turning down... +waiting for node to be active again... +started constructing node... +done. +Operation completed successfully +``` + +### Common Issues and Solutions + +#### Issue: "flush_all failed: failed to connect" +**Solution**: Ensure you're running tests from inside a cluster pod where nodes can reach each other: +```bash +kubectl exec node-0 -- /usr/local/bin/flare-admin ... +``` + +#### Issue: "No proxy node available" +**Solution**: This is expected when all nodes have roles. The test will skip operations requiring proxy nodes. + +#### Issue: "Master command test skipped" +**Solution**: Normal behavior when trying to change an existing master's partition. The validation logic prevents invalid operations. + +## Adding New Tests + +### Mock Server Tests + +Add new tests to `test/e2e/e2e_test.go`: + +```go +func TestNewFeatureE2E(t *testing.T) { + mockServer, err := NewMockFlareServer() + require.NoError(t, err) + defer mockServer.Close() + + flareAdminPath, _ := buildBinaries(t) + + // Add your test logic here +} +``` + +### Kubernetes Tests + +Add new test cases to `scripts/k8s-e2e-test.sh`: + +```bash +# Test X: New feature +echo "Test X: New feature" +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 new-command +echo +``` + +## Continuous Integration + +For CI/CD pipelines: + +```yaml +# GitHub Actions example +- name: Run Mock E2E Tests + run: go test -v ./test/e2e + +- name: Setup Kubernetes + uses: helm/kind-action@v1 + +- name: Deploy Flare Cluster + run: kubectl apply -k flare-cluster-k8s/base + +- name: Run Kubernetes E2E Tests + run: | + ./scripts/copy-to-e2e.sh + ./scripts/k8s-e2e-test.sh +``` + +## Performance Testing + +For load testing, you can run multiple operations: + +```bash +# Stress test with multiple stats calls +for i in {1..100}; do + kubectl exec node-0 -- /usr/local/bin/flare-stats -i flarei.default.svc.cluster.local -p 13300 +done + +# Test concurrent operations +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 stats & +kubectl exec node-0 -- /usr/local/bin/flare-admin -i flarei.default.svc.cluster.local -p 13300 list & +wait +``` + +## Cleanup + +After testing: + +```bash +# Remove flare cluster +kubectl delete -k flare-cluster-k8s/base + +# Clean up local binaries +rm -rf build/ +``` + +## Contributing + +When adding new features: + +1. Add mock server tests for basic functionality +2. Add Kubernetes tests for integration scenarios +3. Update this documentation +4. Ensure all existing tests still pass \ No newline at end of file diff --git a/docs/kubectl-flare.md b/docs/kubectl-flare.md new file mode 100644 index 0000000..3e17b55 --- /dev/null +++ b/docs/kubectl-flare.md @@ -0,0 +1,208 @@ +# kubectl-flare Plugin + +kubectl-flare is a kubectl plugin that allows you to run flare-tools commands directly on the flare index server pod without manually exec'ing into the pod. It automatically finds the index server pod and executes flare-admin or flare-stats commands within the container. + +## Installation + +### Manual Installation + +1. Build and install the plugin: +```bash +./scripts/install-kubectl-plugin.sh +``` + +2. Verify installation: +```bash +kubectl plugin list | grep flare +``` + +### Using Krew (when published) + +```bash +kubectl krew install flare +``` + +## Usage + +The plugin automatically finds the flare index server pod and executes commands on it. + +### Basic Commands + +```bash +# List all nodes +kubectl flare admin list + +# Show cluster statistics +kubectl flare stats nodes + +# Ping the index server +kubectl flare admin ping + +# Ping a specific node +kubectl flare admin ping node-0.flared.default.svc.cluster.local:13301 + +# Add a slave node +kubectl flare admin slave node-2.flared.default.svc.cluster.local:13301 --force + +# Set node as master +kubectl flare admin master node-2.flared.default.svc.cluster.local:13301 --force + +# Reconstruct a node +kubectl flare admin reconstruct node-2.flared.default.svc.cluster.local:13301 --force + +# Show index server status +kubectl flare admin index + +# Dump data +kubectl flare admin dump + +# Show help +kubectl flare --help +kubectl flare admin --help +kubectl flare stats --help +``` + +### Specifying Namespace and Pod + +By default, the plugin looks for the pod `index-0` (using label `statefulset.kubernetes.io/pod-name=index-0`) in the `default` namespace with container `flarei`. + +```bash +# Use a different namespace +kubectl flare -n my-namespace admin list + +# Use a different pod selector +kubectl flare --pod-selector=component=index-server admin list + +# Use a different container name +kubectl flare --container=flarei admin list +``` + +### Command Mapping + +The plugin supports two main command groups: + +1. **admin** - Maps to flare-admin commands + ```bash + kubectl flare admin [args] + # Equivalent to: kubectl exec -- flare-admin [args] + ``` + +2. **stats** - Maps to flare-stats commands + ```bash + kubectl flare stats [args] + # Equivalent to: kubectl exec -- flare-stats [args] + ``` + +### Examples + +```bash +# Check cluster health +kubectl flare admin ping --wait + +# Balance cluster with specific values +kubectl flare admin balance node-0.flared.default.svc.cluster.local:13301:1024 node-1.flared.default.svc.cluster.local:13301:2048 --force + +# Turn down a node +kubectl flare admin down node-2.flared.default.svc.cluster.local:13301 --force + +# Activate a node +kubectl flare admin activate node-2.flared.default.svc.cluster.local:13301 --force + +# Remove a node +kubectl flare admin remove node-2.flared.default.svc.cluster.local:13301 --force + +# Dump keys with partition filter +kubectl flare admin dumpkey --partition 0 + +# Show thread pool status +kubectl flare admin threads + +# Verify cluster configuration +kubectl flare admin verify +``` + +## Troubleshooting + +### Pod Not Found + +If the plugin can't find the index server pod: + +1. Check the pod exists: + ```bash + kubectl get pods | grep index + # or + kubectl get pods -l statefulset.kubernetes.io/pod-name=index-0 + ``` + +2. Use correct namespace: + ```bash + kubectl flare -n correct-namespace admin list + ``` + +3. Use correct pod selector: + ```bash + kubectl get pods --show-labels + kubectl flare --pod-selector=your-label=value admin list + ``` + +### Command Not Found + +If kubectl doesn't recognize the flare plugin: + +1. Ensure the plugin is in PATH: + ```bash + which kubectl-flare + ``` + +2. Check kubectl can find it: + ```bash + kubectl plugin list + ``` + +3. Reinstall the plugin: + ```bash + ./scripts/install-kubectl-plugin.sh + ``` + +## Development + +To build the plugin: +```bash +go build -o kubectl-flare ./cmd/kubectl-flare +``` + +To test locally without installing: +```bash +./kubectl-flare admin list +``` + +## Working Example Output + +```bash +# List all nodes in the cluster +$ kubectl flare admin list +node partition role state balance +node-1.flared.default.svc.cluster.local:13301 0 master active 1 +node-2.flared.default.svc.cluster.local:13301 - proxy active 0 +node-0.flared.default.svc.cluster.local:13301 1 master active 1 + +# Show node statistics +$ kubectl flare stats nodes +hostname:port state role partition balance items conn behind hit size uptime version +node-0.flared.default.svc.cluster.local:13301 active master 1 1 0 16 0 0 0 0s 1.3.4 +node-1.flared.default.svc.cluster.local:13301 active master 0 1 0 17 0 0 0 0s 1.3.4 +node-2.flared.default.svc.cluster.local:13301 active proxy -1 0 0 18 0 0 0 0s 1.3.4 + +# Ping a node +$ kubectl flare admin ping +alive: :13300 +``` + +## Default Configuration + +- **Default namespace**: `default` +- **Default pod selector**: `statefulset.kubernetes.io/pod-name=index-0` +- **Default container**: `flarei` +- **Default index server port**: `13300` (flare-tools v1.0.0+) + +Note: The flare cluster uses port 13300 for the index server (flarei) and port 13301 for data nodes (flared). \ No newline at end of file diff --git a/docs/memacached.protocol.txt b/docs/memacached.protocol.txt new file mode 100644 index 0000000..2c6f0ca --- /dev/null +++ b/docs/memacached.protocol.txt @@ -0,0 +1,1914 @@ +Protocol +-------- + +Clients of memcached communicate with server through TCP connections. +(A UDP interface is also available; details are below under "UDP +protocol.") A given running memcached server listens on some +(configurable) port; clients connect to that port, send commands to +the server, read responses, and eventually close the connection. + +There is no need to send any command to end the session. A client may +just close the connection at any moment it no longer needs it. Note, +however, that clients are encouraged to cache their connections rather +than reopen them every time they need to store or retrieve data. This +is because memcached is especially designed to work very efficiently +with a very large number (many hundreds, more than a thousand if +necessary) of open connections. Caching connections will eliminate the +overhead associated with establishing a TCP connection (the overhead +of preparing for a new connection on the server side is insignificant +compared to this). + +There are two kinds of data sent in the memcache protocol: text lines +and unstructured data. Text lines are used for commands from clients +and responses from servers. Unstructured data is sent when a client +wants to store or retrieve data. The server will transmit back +unstructured data in exactly the same way it received it, as a byte +stream. The server doesn't care about byte order issues in +unstructured data and isn't aware of them. There are no limitations on +characters that may appear in unstructured data; however, the reader +of such data (either a client or a server) will always know, from a +preceding text line, the exact length of the data block being +transmitted. + +Text lines are always terminated by \r\n. Unstructured data is _also_ +terminated by \r\n, even though \r, \n or any other 8-bit characters +may also appear inside the data. Therefore, when a client retrieves +data from a server, it must use the length of the data block (which it +will be provided with) to determine where the data block ends, and not +the fact that \r\n follows the end of the data block, even though it +does. + +Keys +---- + +Data stored by memcached is identified with the help of a key. A key +is a text string which should uniquely identify the data for clients +that are interested in storing and retrieving it. Currently the +length limit of a key is set at 250 characters (of course, normally +clients wouldn't need to use such long keys); the key must not include +control characters or whitespace. + +Commands +-------- + +There are three types of commands. + +Storage commands (there are six: "set", "add", "replace", "append" +"prepend" and "cas") ask the server to store some data identified by a +key. The client sends a command line, and then a data block; after +that the client expects one line of response, which will indicate +success or failure. + +Retrieval commands ("get", "gets", "gat", and "gats") ask the server to +retrieve data corresponding to a set of keys (one or more keys in one +request). The client sends a command line, which includes all the +requested keys; after that for each item the server finds it sends to +the client one response line with information about the item, and one +data block with the item's data; this continues until the server +finished with the "END" response line. + +All other commands don't involve unstructured data. In all of them, +the client sends one command line, and expects (depending on the +command) either one line of response, or several lines of response +ending with "END" on the last line. + +A command line always starts with the name of the command, followed by +parameters (if any) delimited by whitespace. Command names are +lower-case and are case-sensitive. + +Meta Commands +------------- + +Meta commands are a set of new ASCII-based commands. They follow the same +structure of the original commands but have a new flexible feature set. +Meta commands incorporate most features available in the binary protocol, as +well as a flag system to make the commands flexible rather than having a +large number of high level commands. These commands completely replace the +usage of basic Storage and Retrieval commands. + +Meta flags are not to be confused with client bitflags, which is an opaque +number passed by the client. Meta flags change how the command operates, but +they are not stored in cache. + +These work mixed with normal protocol commands on the same connection. All +existing commands continue to work. The meta commands will not replace +specialized commands that do not sit in the Storage or Retrieval categories +noted in the 'Commands' section above. + +All meta commands follow a basic syntax: + + <...>\r\n + +Where is a 2 character command code. + is only for commands with payloads, like set. + +Responses look like: + + <...>\r\n + +Where is a 2 character return code. The number of flags returned are +based off of the flags supplied. + + is only for responses with payloads, with the return code 'VA'. + +Flags are single character codes, ie 'q' or 'k' or 'I', which adjust the +behavior of the command. If a flag requests a response flag (ie 't' for TTL +remaining), it is returned in the same order as they were in the original +command, though this is not strict. + +Flags are single character codes, ie 'q' or 'k' or 'O', which adjust the +behavior of a command. Flags may contain token arguments, which come after the +flag and before the next space or newline, ie 'Oopaque' or 'Kuserkey'. Flags +can return new data or reflect information, in the same order they were +supplied in the request. Sending an 't' flag with a get for an item with 20 +seconds of TTL remaining, would return 't20' in the response. + +All commands accept a tokens 'P' and 'L' which are completely ignored. The +arguments to 'P' and 'L' can be used as hints or path specifications to a +proxy or router inbetween a client and a memcached daemon. For example, a +client may prepend a "path" in the key itself: "mg /path/foo v" or in a proxy +token: "mg foo Lpath/ v" - the proxy may then optionally remove or forward the +token to a memcached daemon, which will ignore them. + +Syntax errors are handled the same as noted under 'Error strings' section +below. + +For usage examples beyond basic syntax, please see the wiki: +https://github.com/memcached/memcached/wiki/MetaCommands + +Expiration times +---------------- + +Some commands involve a client sending some kind of expiration time +(relative to an item or to an operation requested by the client) to +the server. In all such cases, the actual value sent may either be +Unix time (number of seconds since January 1, 1970, as a 32-bit +value), or a number of seconds starting from current time. In the +latter case, this number of seconds may not exceed 60*60*24*30 (number +of seconds in 30 days); if the number sent by a client is larger than +that, the server will consider it to be real Unix time value rather +than an offset from current time. + +Note that a TTL of 1 will sometimes immediately expire. Time is internally +updated on second boundaries, which makes expiration time roughly +/- 1s. +This more proportionally affects very low TTL's. + +Error strings +------------- + +Each command sent by a client may be answered with an error string +from the server. These error strings come in three types: + +- "ERROR\r\n" + + means the client sent a nonexistent command name. + +- "CLIENT_ERROR \r\n" + + means some sort of client error in the input line, i.e. the input + doesn't conform to the protocol in some way. is a + human-readable error string. + +- "SERVER_ERROR \r\n" + + means some sort of server error prevents the server from carrying + out the command. is a human-readable error string. In cases + of severe server errors, which make it impossible to continue + serving the client (this shouldn't normally happen), the server will + close the connection after sending the error line. This is the only + case in which the server closes a connection to a client. + + +In the descriptions of individual commands below, these error lines +are not again specifically mentioned, but clients must allow for their +possibility. + +Authentication +-------------- + +Optional username/password token authentication (see -Y option). Used by +sending a fake "set" command with any key: + +set \r\n +username password\r\n + +key, flags, and exptime are ignored for authentication. Bytes is the length +of the username/password payload. + +- "STORED\r\n" indicates success. After this point any command should work + normally. + +- "CLIENT_ERROR [message]\r\n" will be returned if authentication fails for + any reason. + +Storage commands +---------------- + +First, the client sends a command line which looks like this: + + [noreply]\r\n +cas [noreply]\r\n + +- is "set", "add", "replace", "append" or "prepend" + + "set" means "store this data". + + "add" means "store this data, but only if the server *doesn't* already + hold data for this key". + + "replace" means "store this data, but only if the server *does* + already hold data for this key". + + "append" means "add this data to an existing key after existing data". + + "prepend" means "add this data to an existing key before existing data". + + The append and prepend commands do not accept flags or exptime. + They update existing data portions, and ignore new flag and exptime + settings. + + "cas" is a check and set operation which means "store this data but + only if no one else has updated since I last fetched it." + +- is the key under which the client asks to store the data + +- is an arbitrary 16-bit unsigned integer (written out in + decimal) that the server stores along with the data and sends back + when the item is retrieved. Clients may use this as a bit field to + store data-specific information; this field is opaque to the server. + Note that in memcached 1.2.1 and higher, flags may be 32-bits, instead + of 16, but you might want to restrict yourself to 16 bits for + compatibility with older versions. + +- is expiration time. If it's 0, the item never expires + (although it may be deleted from the cache to make place for other + items). If it's non-zero (either Unix time or offset in seconds from + current time), it is guaranteed that clients will not be able to + retrieve this item after the expiration time arrives (measured by + server time). If a negative value is given the item is immediately + expired. + +- is the number of bytes in the data block to follow, *not* + including the delimiting \r\n. may be zero (in which case + it's followed by an empty data block). + +- is a unique 64-bit value of an existing entry. + Clients should use the value returned from the "gets" command + when issuing "cas" updates. + +- "noreply" optional parameter instructs the server to not send the + reply. NOTE: if the request line is malformed, the server can't + parse "noreply" option reliably. In this case it may send the error + to the client, and not reading it on the client side will break + things. Client should construct only valid requests. + +After this line, the client sends the data block: + +\r\n + +- is a chunk of arbitrary 8-bit data of length + from the previous line. + +After sending the command line and the data block the client awaits +the reply, which may be: + +- "STORED\r\n", to indicate success. + +- "NOT_STORED\r\n" to indicate the data was not stored, but not +because of an error. This normally means that the +condition for an "add" or a "replace" command wasn't met. + +- "EXISTS\r\n" to indicate that the item you are trying to store with +a "cas" command has been modified since you last fetched it. + +- "NOT_FOUND\r\n" to indicate that the item you are trying to store +with a "cas" command did not exist. + + +Retrieval command: +------------------ + +The retrieval commands "get" and "gets" operate like this: + +get *\r\n +gets *\r\n + +- * means one or more key strings separated by whitespace. + +After this command, the client expects zero or more items, each of +which is received as a text line followed by a data block. After all +the items have been transmitted, the server sends the string + +"END\r\n" + +to indicate the end of response. + +Each item sent by the server looks like this: + +VALUE []\r\n +\r\n + +- is the key for the item being sent + +- is the flags value set by the storage command + +- is the length of the data block to follow, *not* including + its delimiting \r\n + +- is a unique 64-bit integer that uniquely identifies + this specific item. + +- is the data for this item. + +If some of the keys appearing in a retrieval request are not sent back +by the server in the item list this means that the server does not +hold items with such keys (because they were never stored, or stored +but deleted to make space for more items, or expired, or explicitly +deleted by a client). + + +Deletion +-------- + +The command "delete" allows for explicit deletion of items: + +delete [noreply]\r\n + +- is the key of the item the client wishes the server to delete + +- "noreply" optional parameter instructs the server to not send the + reply. See the note in Storage commands regarding malformed + requests. + +The response line to this command can be one of: + +- "DELETED\r\n" to indicate success + +- "NOT_FOUND\r\n" to indicate that the item with this key was not + found. + +See the "flush_all" command below for immediate invalidation +of all existing items. + + +Increment/Decrement +------------------- + +Commands "incr" and "decr" are used to change data for some item +in-place, incrementing or decrementing it. The data for the item is +treated as decimal representation of a 64-bit unsigned integer. If +the current data value does not conform to such a representation, the +incr/decr commands return an error (memcached <= 1.2.6 treated the +bogus value as if it were 0, leading to confusion). Also, the item +must already exist for incr/decr to work; these commands won't pretend +that a non-existent key exists with value 0; instead, they will fail. + +The client sends the command line: + +incr [noreply]\r\n + +or + +decr [noreply]\r\n + +- is the key of the item the client wishes to change + +- is the amount by which the client wants to increase/decrease +the item. It is a decimal representation of a 64-bit unsigned integer. + +- "noreply" optional parameter instructs the server to not send the + reply. See the note in Storage commands regarding malformed + requests. + +The response will be one of: + +- "NOT_FOUND\r\n" to indicate the item with this value was not found + +- \r\n , where is the new value of the item's data, + after the increment/decrement operation was carried out. + +Note that underflow in the "decr" command is caught: if a client tries +to decrease the value below 0, the new value will be 0. Overflow in +the "incr" command will wrap around the 64 bit mark. + +Note also that decrementing a number such that it loses length isn't +guaranteed to decrement its returned length. The number MAY be +space-padded at the end, but this is purely an implementation +optimization, so you also shouldn't rely on that. + +Touch +----- + +The "touch" command is used to update the expiration time of an existing item +without fetching it. + +touch [noreply]\r\n + +- is the key of the item the client wishes the server to touch + +- is expiration time. Works the same as with the update commands + (set/add/etc). This replaces the existing expiration time. If an existing + item were to expire in 10 seconds, but then was touched with an + expiration time of "20", the item would then expire in 20 seconds. + +- "noreply" optional parameter instructs the server to not send the + reply. See the note in Storage commands regarding malformed + requests. + +The response line to this command can be one of: + +- "TOUCHED\r\n" to indicate success + +- "NOT_FOUND\r\n" to indicate that the item with this key was not + found. + +Get And Touch +------------- + +The "gat" and "gats" commands are used to fetch items and update the +expiration time of an existing items. + +gat *\r\n +gats *\r\n + +- is expiration time. + +- * means one or more key strings separated by whitespace. + +After this command, the client expects zero or more items, each of +which is received as a text line followed by a data block. After all +the items have been transmitted, the server sends the string + +"END\r\n" + +to indicate the end of response. + +Each item sent by the server looks like this: + +VALUE []\r\n +\r\n + +- is the key for the item being sent + +- is the flags value set by the storage command + +- is the length of the data block to follow, *not* including + its delimiting \r\n + +- is a unique 64-bit integer that uniquely identifies + this specific item. + +- is the data for this item. + +Meta Debug +---------- + +The meta debug command is a human readable dump of all available internal +metadata of an item, minus the value. + +me \r\n + +- means one key string. +- if is 'b', then is a base64 encoded binary value. the response + key will also be base64 encoded. + +The response looks like: + +ME =*\r\n + +A miss looks like: + +EN\r\n + +Each of the keys and values are the internal data for the item. + +exp = expiration time +la = time in seconds since last access +cas = CAS ID +fetch = whether an item has been fetched before +cls = slab class id +size = total size in bytes + +Others may be added. + +Meta Get +-------- + +The meta get command is the generic command for retrieving key data from +memcached. Based on the flags supplied, it can replace all of the commands: +"get", "gets", "gat", "gats", "touch", as well as adding new options. + +mg *\r\n + +- means one key string. Unlike "get" metaget can only take a single key. + +- are a set of single character codes ended with a space or newline. + flags may have token strings after the initial character. + +After this command, the client expects an item to be returned, received as a +text line followed by an optional data block. + +If a response line appearing in a retrieval request is not sent back +by the server this means that the server does not +have the item (because it was never stored, or stored +but deleted to make space for more items, or expired, or explicitly +deleted by a client). + +An item sent by the server looks like: + +VA *\r\n +\r\n + +- is the size of in bytes, minus the \r\n + +- * are flags returned by the server, based on the command flags. + They are added in order specified by the flags sent. + +- is the data for this item. Note that the data block is + optional, requiring the 'v' flag to be supplied. + +If the request did not ask for a value in the response (v) flag, the server +response looks like: + +HD *\r\n + +If the request resulted in a miss, the response looks like: + +EN\r\n + +Unless the (q) flag was supplied, which suppresses the status code for a miss. + +The flags used by the 'mg' command are: + +- b: interpret key as base64 encoded binary value +- c: return item cas token +- f: return client flags token +- h: return whether item has been hit before as a 0 or 1 +- k: return key as a token +- l: return time since item was last accessed in seconds +- O(token): opaque value, consumes a token and copies back with response +- q: use noreply semantics for return codes. +- s: return item size token +- t: return item TTL remaining in seconds (-1 for unlimited) +- u: don't bump the item in the LRU +- v: return item value in + +These flags can modify the item: +- E(token): use token as new CAS value if item is modified +- N(token): vivify on miss, takes TTL as a argument +- R(token): if remaining TTL is less than token, win for recache +- T(token): update remaining TTL + +These extra flags can be added to the response: +- W: client has "won" the recache flag +- X: item is stale +- Z: item has already sent a winning flag + +The flags are now repeated with detailed information where useful: + +- b: interpret key as base64 encoded binary value + +This flag instructs memcached to run a base64 decoder on before looking +it up. This allows storing and fetching of binary packed keys, so long as they +are sent to memcached in base64 encoding. + +If 'b' flag is sent in the response, and a key is returned via 'k', this +signals to the client that the key is base64 encoded binary. + +- h: return whether item has been hit before as a 0 or 1 +- l: return time since item was last accessed in seconds + +The above two flags return the value of "hit before?" and "last access time" +before the command was processed. Otherwise this would always show a 1 for +hit or always show an access time of "0" unless combined with the "u" flag. + +- O(token): opaque value, consumes a token and copies back with response + +The O(opaque) token is used by this and other commands to allow easier +pipelining of requests while saving bytes on the wire for responses. For +example: if pipelining three get commands together, you may not know which +response belongs to which without also retrieving the key. If the key is very +long this can generate a lot of traffic, especially if the data block is very +small. Instead, you can supply an "O" flag for each mg with tokens of "1" "2" +and "3", to match up responses to the request. + +Opaque tokens may be up to 32 bytes in length, and are a string similar to +keys. + +- q: use noreply semantics for return codes. + +Noreply is a method of reducing the amount of data sent back by memcached to +the client for normal responses. In the case of metaget, if an item is not +available the "VA" line is normally not sent, and the response is terminated by +"EN\r\n". + +With noreply enabled, the "EN\r\n" marker is suppressed. This allows you to +pipeline several mg's together and read all of the responses without the +"EN\r\n" lines in the middle. + +Errors are always returned. + +- u: don't bump the item in the LRU + +It is possible to access an item without causing it to be "bumped" to the head +of the LRU. This also avoids marking an item as being hit or updating its last +access time. + +- v: return item value in + +The data block for a metaget response is optional, requiring this flag to be +passed in. The response code also changes from "HD" to "VA " + +These flags can modify the item: +- E(token): use token as new CAS value if item is modified + +Normally when an item is created it is given a CAS value from an internal +atomically incrementing counter. This allows overriding the CAS (8 byte +unsigned integer) with the specified value. This is useful for using an +external system to version cache data (row versions, clocks, etc). + +- N(token): vivify on miss, takes TTL as a argument + +Used to help with so called "dog piling" problems with recaching of popular +items. If supplied, and metaget does not find the item in cache, it will +create a stub item with the key and TTL as supplied. If such an item is +created a 'W' flag is added to the response to indicate to a client that they +have "won" the right to recache an item. + +The automatically created item has 0 bytes of data. + +Further requests will see a 'Z' flag to indicate that another client has +already received the win flag. + +Can be combined with CAS flags to gate the update further. + +- R(token): if token is less than remaining TTL win for recache + +Similar to and can be combined with 'N'. If the remaining TTL of an item is +below the supplied token, return a 'W' flag to indicate the client has "won" +the right to recache an item. This allows refreshing an item before it leads to +a miss. + +- T(token): update remaining TTL + +Similar to "touch" and "gat" commands, updates the remaining TTL of an item if +hit. + +These extra flags can be added to the response: +- W: client has "won" the recache flag + +When combined with N or R flags, a client may be informed they have "won" +ownership of a cache item. This allows a single client to be atomically +notified that it should cache or update an item. Further clients requesting +the item can use the existing data block (if valid or stale), retry, wait, or +take some other action while the item is missing. + +This is used when the act of recaching an item can cause undue load on another +system (CPU, database accesses, time, and so on). + +- X: item is stale + +Items can be marked as stale by the metadelete command. This indicates to the +client that an object needs to be updated, and that it should make a decision +o if potentially stale data is safe to use. + +This is used when you want to convince clients to recache an item, but it's +safe to use pre-existing data while the recache happens. + +- Z: item has already sent a winning flag + +When combined with the X flag, or the client N or R flags, this extra response +flag indicates to a client that a different client is responsible for +recaching this item. If there is data supplied it may use it, or the client +may decide to retry later or take some other action. + +Meta Set +-------- + +The meta set command a generic command for storing data to memcached. Based +on the flags supplied, it can replace all storage commands (see token M) as +well as adds new options. + +ms *\r\n + +- means one key string. + +- is the length of the payload data. + +- are a set of single character codes ended with a space or newline. + flags may have strings after the initial character. + +After this line, the client sends the data block: + +\r\n + +- is a chunk of arbitrary 8-bit data of length supplied by an 'S' + flag and token from the request line. If no 'S' flag is supplied the data + is assumed to be 0 length. + +After sending the command line and the data block the client awaits +the reply, which is of the format: + + *\r\n + +Where CD is one of: + +- "HD" (STORED), to indicate success. + +- "NS" (NOT_STORED), to indicate the data was not stored, but not +because of an error. + +- "EX" (EXISTS), to indicate that the item you are trying to store with +CAS semantics has been modified since you last fetched it. + +- "NF" (NOT_FOUND), to indicate that the item you are trying to store +with CAS semantics did not exist. + +The flags used by the 'ms' command are: + +- b: interpret key as base64 encoded binary value (see metaget) +- c: return CAS value if successfully stored. +- C(token): compare CAS value when storing item +- E(token): use token as new CAS value (see metaget for detail) +- F(token): set client flags to token (32 bit unsigned numeric) +- I: invalidate. set-to-invalid if supplied CAS is older than item's CAS +- k: return key as a token +- O(token): opaque value, consumes a token and copies back with response +- q: use noreply semantics for return codes +- s: return the size of the stored item on success (ie; new size on append) +- T(token): Time-To-Live for item, see "Expiration" above. +- M(token): mode switch to change behavior to add, replace, append, prepend +- N(token): if in append mode, autovivify on miss with supplied TTL + +The flags are now repeated with detailed information where useful: + +- c: returns the CAS value on successful storage. Will return 0 on error, but + clients must not depend on the return value being zero. In future versions +this may return the current CAS value for "EX" return code conditions. + +- C(token): compare CAS value when storing item + +Similar to the basic "cas" command, only store item if the supplied token +matches the current CAS value of the item. When combined with the 'I' flag, a +CAS value that is _lower_ than the current value may be accepted, but the item +will be marked as "stale", returning the X flag with mget requests. + +- F(token): set client flags to token (32 bit unsigned numeric) + +Sets flags to 0 if not supplied. + +- I: invalid. set-to-invalid if CAS is older than it should be. + +Functional when combined with 'C' flag above. + +- O(token): opaque value, consumes a token and copies back with response + +See description under 'Meta Get' + +- q: use noreply semantics for return codes + +Noreply is a method of reducing the amount of data sent back by memcached to +the client for normal responses. In the case of metaset, a response that +would start with "HD" will not be sent. Any other code, such as "EX" +(EXISTS) will still be returned. + +Errors are always returned. + +- M(token): mode switch. Takes a single character for the mode. + +E: "add" command. LRU bump and return NS if item exists. Else +add. +A: "append" command. If item exists, append the new value to its data. +P: "prepend" command. If item exists, prepend the new value to its data. +R: "replace" command. Set only if item already exists. +S: "set" command. The default mode, added for completeness. + +The "cas" command is supplanted by specifying the cas value with the 'C' flag. +Append and Prepend modes will also respect a supplied cas value. + +- N(token): if in append mode, autovivify on miss with supplied TTL + +Append and Prepend modes normally ignore the T argument, as they cannot create +a new item on a miss. If N is supplied, and append reaches a miss, it will +create a new item seeded with the data from the append command. It uses the +TTL from N instead of T to be consistent with the usage of N in other +commands. + +Meta Delete +----------- + +The meta delete command allows for explicit deletion of items, as well as +marking items as "stale" to allow serving items as stale during revalidation. + +md *\r\n + +- means one key string. + +- are a set of single character codes ended with a space or newline. + flags may have strings after the initial character. + +The response is in the format: + + *\r\n + +Where CD is one of: + +- "HD" (DELETED), to indicate success + +- "NF" (NOT_FOUND), to indicate that the item with this key was not found. + +- "EX" (EXISTS), to indicate that the supplied CAS token does not match the + stored item. + +The flags used by the 'md' command are: + +- b: interpret key as base64 encoded binary value (see metaget) +- C(token): compare CAS value +- E(token): use token as new CAS value (see metaget for detail) +- I: invalidate. mark as stale, bumps CAS. +- k: return key +- O(token): opaque to copy back. +- q: noreply +- T(token): updates TTL, only when paired with the 'I' flag +- x: removes the item value, but leaves the item. + +The flags are now repeated with detailed information where useful: + +- C(token): compare CAS value + +Can be used to only delete or mark-stale if a supplied CAS value matches. + +- I: invalidate. mark as stale, bumps CAS. + +Instead of removing an item, this will give the item a new CAS value and mark +it as stale. This means when it is later fetched by metaget, the client will +be supplied an 'X' flag to show the data is stale and needs to be recached. + +- O(token): opaque to copy back. + +See description under 'Meta Get' + +- q: noreply + +See description under 'Meta Set'. In the case of meta delete, this will hide +response lines with the code "DE". It will still return any other responses. + +- T(token): updates TTL, only when paired with the 'I' flag + +When marking an item as stale with 'I', the 'T' flag can be used to update the +TTL as well; limiting the amount of time an item will live while stale and +waiting to be recached. + +- x: removes the item value, but leaves the item. + +This deletes the value off of an item (by replacing it with an empty value +item atomically). Combined with I this can leave what is effectively a +tombstone of a previous value. + +Meta Arithmetic +--------------- + +The meta arithmetic command allows for basic operations against numerical +values. This replaces the "incr" and "decr" commands. Values are unsigned +64bit integers. Decrementing will reach 0 rather than underflow. Incrementing +can overflow. + +ma *\r\n + +- means one key string. + +- are a set of single character codes ended with a space or newline. + flags may have strings after the initial character. + +The response is in the format: + + *\r\n + +Where CD is one of: + +- "HD" to indicate success + +- "NF" (NOT_FOUND), to indicate that the item with this key was not found. + +- "NS" (NOT_STORED), to indicate that the item was not created as requested + after a miss. + +- "EX" (EXISTS), to indicate that the supplied CAS token does not match the + stored item. + +If the 'v' flag is supplied, the response is formatted as: + +VA *\r\n +\r\n + +The flags used by the 'ma' command are: + +- b: interpret key as base64 encoded binary value (see metaget) +- C(token): compare CAS value (see mset) +- E(token): use token as new CAS value (see metaget for detail) +- N(token): auto create item on miss with supplied TTL +- J(token): initial value to use if auto created after miss (default 0) +- D(token): delta to apply (decimal unsigned 64-bit number, default 1) +- T(token): update TTL on success +- M(token): mode switch to change between incr and decr modes. +- O(token): opaque value, consumes a token and copies back with response +- q: use noreply semantics for return codes (see details under mset) +- t: return current TTL +- c: return current CAS value if successful. +- v: return new value +- k: return key as a token + +The flags are now repeated with detailed information where useful: + +- C(token): compare CAS value + +Can be used to only incr/decr if a supplied CAS value matches. A new CAS value +is generated after a delta is applied. Add the 'c' flag to get the new CAS +value after success. + +- N(token): auto create item on miss with supplied TTL + +Similar to mget, on a miss automatically create the item. A value can be +seeded using the J flag. + +- J(token): initial value + +An unsigned 64-bit integer which will be seeded as the value on a miss. Must be +combined with an N flag. + +- D(token): delta to apply + +An unsigned integer to either add or subtract from the currently stored +number. + +- T(token): update TTL + +On success, sets the remaining TTL to the supplied value. + +-M(token): mode switch. Takes a single character for the mode. + +I: Increment mode (default) ++: Alias for increment +D: Decrement mode +-: Alias for decrement + +Meta No-Op +---------- + +The meta no-op command exists solely to return a static response code. It +takes no flags, no arguments. + +"mn\r\n" + +This returns the static response: + +"MN\r\n" + +This command is useful when used with the 'q' flag and pipelining commands. +For example, with 'mg' the response lines are blank on miss when the 'q' flag +is supplied. If pipelining several 'mg's together with noreply semantics, an +"mn\r\n" command can be tagged to the end of the chain, which will return an +"MN\r\n", signalling to a client that all previous commands have been +processed. + +Slabs Reassign +-------------- + +NOTE: This command is subject to change as of this writing. + +The slabs reassign command is used to redistribute memory once a running +instance has hit its limit. It might be desirable to have memory laid out +differently than was automatically assigned after the server started. + +slabs reassign \r\n + +- is an id number for the slab class to steal a page from + +A source class id of -1 means "pick from any valid class" + +- is an id number for the slab class to move a page to + +The response line could be one of: + +- "OK" to indicate the page has been scheduled to move + +- "BUSY [message]" to indicate a page is already being processed, try again + later. + +- "BADCLASS [message]" a bad class id was specified + +- "NOSPARE [message]" source class has no spare pages + +- "NOTFULL [message]" dest class must be full to move new pages to it + +- "UNSAFE [message]" source class cannot move a page right now + +- "SAME [message]" must specify different source/dest ids. + +Slabs Automove +-------------- + +NOTE: This command is subject to change as of this writing. + +The slabs automove command enables a background thread which decides on its +own when to move memory between slab classes. Its implementation and options +will likely be in flux for several versions. See the wiki/mailing list for +more details. + +The automover can be enabled or disabled at runtime with this command. + +slabs automove <0|1|2> + +- 0|1|2 is the indicator on whether to enable the slabs automover or not. + +The response should always be "OK\r\n" + +- <0> means to set the thread on standby + +- <1> means to return pages to a global pool when there are more than 2 pages + worth of free chunks in a slab class. Pages are then re-assigned back into + other classes as-needed. + +- <2> is a highly aggressive mode which causes pages to be moved every time + there is an eviction. It is not recommended to run for very long in this + mode unless your access patterns are very well understood. + +LRU Tuning +---------- + +Memcached supports multiple LRU algorithms, with a few tunables. Effort is +made to have sane defaults however you are able to tune while the daemon is +running. + +The traditional model is "flat" mode, which is a single LRU chain per slab +class. The newer (with `-o modern` or `-o lru_maintainer`) is segmented into +HOT, WARM, COLD. There is also a TEMP LRU. See doc/new_lru.txt for details. + +lru